karafka 2.5.5 → 2.5.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (215) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +20 -0
  3. data/LICENSE-COMM +4 -0
  4. data/README.md +2 -2
  5. data/certs/expired.txt +2 -0
  6. data/karafka.gemspec +23 -23
  7. data/lib/active_job/karafka.rb +2 -2
  8. data/lib/active_job/queue_adapters/karafka_adapter.rb +5 -5
  9. data/lib/karafka/active_job/consumer.rb +3 -3
  10. data/lib/karafka/active_job/current_attributes.rb +4 -4
  11. data/lib/karafka/active_job/job_options_contract.rb +2 -2
  12. data/lib/karafka/admin/acl.rb +3 -3
  13. data/lib/karafka/admin/configs/resource.rb +1 -1
  14. data/lib/karafka/admin/configs.rb +1 -1
  15. data/lib/karafka/admin/consumer_groups.rb +8 -8
  16. data/lib/karafka/admin/contracts/replication.rb +2 -2
  17. data/lib/karafka/admin/replication.rb +21 -21
  18. data/lib/karafka/admin/topics.rb +6 -6
  19. data/lib/karafka/admin.rb +4 -5
  20. data/lib/karafka/app.rb +3 -3
  21. data/lib/karafka/base_consumer.rb +34 -30
  22. data/lib/karafka/cli/base.rb +8 -8
  23. data/lib/karafka/cli/console.rb +1 -1
  24. data/lib/karafka/cli/contracts/server.rb +12 -12
  25. data/lib/karafka/cli/help.rb +2 -2
  26. data/lib/karafka/cli/info.rb +4 -4
  27. data/lib/karafka/cli/install.rb +11 -11
  28. data/lib/karafka/cli/server.rb +6 -6
  29. data/lib/karafka/cli/swarm.rb +1 -1
  30. data/lib/karafka/cli/topics/align.rb +4 -4
  31. data/lib/karafka/cli/topics/base.rb +5 -5
  32. data/lib/karafka/cli/topics/create.rb +2 -2
  33. data/lib/karafka/cli/topics/delete.rb +2 -2
  34. data/lib/karafka/cli/topics/help.rb +5 -1
  35. data/lib/karafka/cli/topics/plan.rb +16 -16
  36. data/lib/karafka/cli/topics/repartition.rb +3 -3
  37. data/lib/karafka/cli/topics.rb +22 -22
  38. data/lib/karafka/cli.rb +2 -2
  39. data/lib/karafka/connection/client.rb +17 -17
  40. data/lib/karafka/connection/listener.rb +6 -6
  41. data/lib/karafka/connection/mode.rb +1 -1
  42. data/lib/karafka/connection/proxy.rb +1 -1
  43. data/lib/karafka/connection/status.rb +2 -2
  44. data/lib/karafka/constraints.rb +3 -3
  45. data/lib/karafka/embedded.rb +3 -3
  46. data/lib/karafka/env.rb +4 -4
  47. data/lib/karafka/errors.rb +6 -1
  48. data/lib/karafka/execution_mode.rb +1 -1
  49. data/lib/karafka/helpers/config_importer.rb +2 -2
  50. data/lib/karafka/helpers/interval_runner.rb +4 -2
  51. data/lib/karafka/helpers/multi_delegator.rb +1 -1
  52. data/lib/karafka/instrumentation/assignments_tracker.rb +9 -9
  53. data/lib/karafka/instrumentation/callbacks/error.rb +5 -5
  54. data/lib/karafka/instrumentation/callbacks/oauthbearer_token_refresh.rb +4 -4
  55. data/lib/karafka/instrumentation/callbacks/rebalance.rb +6 -6
  56. data/lib/karafka/instrumentation/callbacks/statistics.rb +5 -5
  57. data/lib/karafka/instrumentation/logger.rb +7 -7
  58. data/lib/karafka/instrumentation/logger_listener.rb +76 -63
  59. data/lib/karafka/instrumentation/vendors/appsignal/base.rb +1 -1
  60. data/lib/karafka/instrumentation/vendors/appsignal/client.rb +1 -1
  61. data/lib/karafka/instrumentation/vendors/appsignal/errors_listener.rb +1 -1
  62. data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb +36 -36
  63. data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +33 -28
  64. data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +38 -38
  65. data/lib/karafka/instrumentation/vendors/kubernetes/base_listener.rb +5 -5
  66. data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +1 -1
  67. data/lib/karafka/instrumentation/vendors/kubernetes/swarm_liveness_listener.rb +1 -1
  68. data/lib/karafka/licenser.rb +115 -8
  69. data/lib/karafka/messages/builders/batch_metadata.rb +4 -2
  70. data/lib/karafka/messages/messages.rb +1 -1
  71. data/lib/karafka/patches/rdkafka/bindings.rb +2 -2
  72. data/lib/karafka/pro/active_job/job_options_contract.rb +2 -2
  73. data/lib/karafka/pro/cleaner/messages/messages.rb +10 -0
  74. data/lib/karafka/pro/cli/contracts/server.rb +12 -12
  75. data/lib/karafka/pro/cli/parallel_segments/base.rb +4 -4
  76. data/lib/karafka/pro/cli/parallel_segments/collapse.rb +5 -5
  77. data/lib/karafka/pro/cli/parallel_segments/distribute.rb +3 -3
  78. data/lib/karafka/pro/cli/parallel_segments.rb +7 -7
  79. data/lib/karafka/pro/cli/topics/health.rb +162 -0
  80. data/lib/karafka/pro/cli/topics.rb +52 -0
  81. data/lib/karafka/pro/connection/manager.rb +14 -14
  82. data/lib/karafka/pro/encryption/contracts/config.rb +2 -2
  83. data/lib/karafka/pro/encryption/messages/middleware.rb +2 -2
  84. data/lib/karafka/pro/encryption/messages/parser.rb +2 -2
  85. data/lib/karafka/pro/encryption/setup/config.rb +2 -2
  86. data/lib/karafka/pro/iterator/tpl_builder.rb +2 -2
  87. data/lib/karafka/pro/iterator.rb +1 -1
  88. data/lib/karafka/pro/loader.rb +2 -1
  89. data/lib/karafka/pro/processing/adaptive_iterator/consumer.rb +1 -1
  90. data/lib/karafka/pro/processing/coordinators/virtual_offset_manager.rb +24 -14
  91. data/lib/karafka/pro/processing/filters/base.rb +1 -1
  92. data/lib/karafka/pro/processing/filters/delayer.rb +2 -2
  93. data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb +1 -1
  94. data/lib/karafka/pro/processing/offset_metadata/consumer.rb +1 -1
  95. data/lib/karafka/pro/processing/parallel_segments/filters/base.rb +6 -6
  96. data/lib/karafka/pro/processing/partitioner.rb +3 -3
  97. data/lib/karafka/pro/processing/periodic_job/consumer.rb +6 -5
  98. data/lib/karafka/pro/processing/piping/consumer.rb +7 -7
  99. data/lib/karafka/pro/processing/schedulers/base.rb +5 -5
  100. data/lib/karafka/pro/processing/schedulers/default.rb +5 -5
  101. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb +6 -3
  102. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb +6 -3
  103. data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb +6 -3
  104. data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb +2 -2
  105. data/lib/karafka/pro/processing/strategies/default.rb +22 -22
  106. data/lib/karafka/pro/processing/strategies/dlq/default.rb +7 -7
  107. data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb +6 -3
  108. data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb +6 -3
  109. data/lib/karafka/pro/processing/strategies/ftr/default.rb +2 -2
  110. data/lib/karafka/pro/processing/strategies/lrj/default.rb +2 -2
  111. data/lib/karafka/pro/processing/strategies/lrj/ftr.rb +6 -3
  112. data/lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb +6 -3
  113. data/lib/karafka/pro/processing/strategies/lrj/mom.rb +2 -2
  114. data/lib/karafka/pro/recurring_tasks/consumer.rb +2 -2
  115. data/lib/karafka/pro/recurring_tasks/contracts/config.rb +2 -2
  116. data/lib/karafka/pro/recurring_tasks/contracts/task.rb +2 -2
  117. data/lib/karafka/pro/recurring_tasks/dispatcher.rb +2 -2
  118. data/lib/karafka/pro/recurring_tasks/listener.rb +1 -1
  119. data/lib/karafka/pro/recurring_tasks/matcher.rb +2 -2
  120. data/lib/karafka/pro/recurring_tasks/serializer.rb +5 -5
  121. data/lib/karafka/pro/recurring_tasks/setup/config.rb +3 -3
  122. data/lib/karafka/pro/recurring_tasks/task.rb +4 -4
  123. data/lib/karafka/pro/recurring_tasks.rb +4 -4
  124. data/lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb +2 -2
  125. data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +2 -2
  126. data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb +1 -1
  127. data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb +2 -2
  128. data/lib/karafka/pro/routing/features/direct_assignments/contracts/consumer_group.rb +2 -2
  129. data/lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb +2 -2
  130. data/lib/karafka/pro/routing/features/direct_assignments/topic.rb +1 -1
  131. data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb +2 -2
  132. data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb +2 -2
  133. data/lib/karafka/pro/routing/features/inline_insights/contracts/topic.rb +2 -2
  134. data/lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb +2 -2
  135. data/lib/karafka/pro/routing/features/long_running_job/topic.rb +1 -1
  136. data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb +2 -2
  137. data/lib/karafka/pro/routing/features/multiplexing.rb +5 -5
  138. data/lib/karafka/pro/routing/features/non_blocking_job/topic.rb +1 -1
  139. data/lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb +2 -2
  140. data/lib/karafka/pro/routing/features/offset_metadata/topic.rb +1 -1
  141. data/lib/karafka/pro/routing/features/offset_metadata.rb +1 -1
  142. data/lib/karafka/pro/routing/features/parallel_segments/consumer_group.rb +5 -5
  143. data/lib/karafka/pro/routing/features/parallel_segments/contracts/consumer_group.rb +2 -2
  144. data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +2 -2
  145. data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb +3 -3
  146. data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +2 -2
  147. data/lib/karafka/pro/routing/features/patterns/topic.rb +1 -1
  148. data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb +2 -2
  149. data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb +2 -2
  150. data/lib/karafka/pro/routing/features/periodic_job/topic.rb +1 -1
  151. data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb +7 -7
  152. data/lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb +2 -2
  153. data/lib/karafka/pro/routing/features/scheduled_messages/builder.rb +13 -13
  154. data/lib/karafka/pro/routing/features/scheduled_messages/contracts/topic.rb +2 -2
  155. data/lib/karafka/pro/routing/features/swarm/contracts/routing.rb +2 -2
  156. data/lib/karafka/pro/routing/features/swarm/contracts/topic.rb +2 -2
  157. data/lib/karafka/pro/routing/features/swarm.rb +1 -1
  158. data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb +2 -2
  159. data/lib/karafka/pro/routing/features/virtual_partitions/config.rb +7 -7
  160. data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +2 -2
  161. data/lib/karafka/pro/scheduled_messages/consumer.rb +4 -4
  162. data/lib/karafka/pro/scheduled_messages/contracts/config.rb +2 -2
  163. data/lib/karafka/pro/scheduled_messages/contracts/message.rb +10 -10
  164. data/lib/karafka/pro/scheduled_messages/daily_buffer.rb +2 -2
  165. data/lib/karafka/pro/scheduled_messages/deserializers/headers.rb +4 -4
  166. data/lib/karafka/pro/scheduled_messages/dispatcher.rb +5 -5
  167. data/lib/karafka/pro/scheduled_messages/proxy.rb +8 -8
  168. data/lib/karafka/pro/scheduled_messages/schema_validator.rb +1 -1
  169. data/lib/karafka/pro/scheduled_messages/setup/config.rb +2 -2
  170. data/lib/karafka/pro/scheduled_messages/state.rb +1 -1
  171. data/lib/karafka/pro/scheduled_messages/tracker.rb +2 -2
  172. data/lib/karafka/pro/scheduled_messages.rb +2 -2
  173. data/lib/karafka/pro/swarm/liveness_listener.rb +2 -2
  174. data/lib/karafka/process.rb +1 -1
  175. data/lib/karafka/processing/coordinator.rb +1 -1
  176. data/lib/karafka/processing/inline_insights/consumer.rb +4 -4
  177. data/lib/karafka/processing/inline_insights/tracker.rb +6 -6
  178. data/lib/karafka/processing/jobs/base.rb +6 -4
  179. data/lib/karafka/processing/jobs_queue.rb +10 -0
  180. data/lib/karafka/processing/schedulers/default.rb +4 -4
  181. data/lib/karafka/processing/strategies/base.rb +6 -6
  182. data/lib/karafka/processing/strategies/default.rb +13 -13
  183. data/lib/karafka/processing/strategies/dlq.rb +1 -1
  184. data/lib/karafka/processing/worker.rb +5 -5
  185. data/lib/karafka/railtie.rb +11 -11
  186. data/lib/karafka/routing/builder.rb +3 -3
  187. data/lib/karafka/routing/contracts/consumer_group.rb +6 -6
  188. data/lib/karafka/routing/contracts/routing.rb +2 -2
  189. data/lib/karafka/routing/contracts/topic.rb +4 -4
  190. data/lib/karafka/routing/features/active_job/contracts/topic.rb +3 -3
  191. data/lib/karafka/routing/features/base/expander.rb +4 -4
  192. data/lib/karafka/routing/features/base.rb +8 -8
  193. data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +2 -2
  194. data/lib/karafka/routing/features/declaratives/contracts/topic.rb +2 -2
  195. data/lib/karafka/routing/features/deserializers/contracts/topic.rb +2 -2
  196. data/lib/karafka/routing/features/eofed/contracts/topic.rb +3 -3
  197. data/lib/karafka/routing/features/inline_insights/contracts/topic.rb +2 -2
  198. data/lib/karafka/routing/features/inline_insights.rb +7 -7
  199. data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb +2 -2
  200. data/lib/karafka/routing/subscription_group.rb +9 -9
  201. data/lib/karafka/runner.rb +3 -3
  202. data/lib/karafka/server.rb +14 -5
  203. data/lib/karafka/setup/attributes_map.rb +7 -7
  204. data/lib/karafka/setup/config.rb +11 -11
  205. data/lib/karafka/setup/contracts/config.rb +2 -2
  206. data/lib/karafka/setup/defaults_injector.rb +11 -11
  207. data/lib/karafka/swarm/manager.rb +6 -6
  208. data/lib/karafka/swarm/node.rb +8 -37
  209. data/lib/karafka/swarm/producer_replacer.rb +110 -0
  210. data/lib/karafka/swarm/supervisor.rb +9 -6
  211. data/lib/karafka/swarm.rb +1 -1
  212. data/lib/karafka/time_trackers/pause.rb +1 -1
  213. data/lib/karafka/version.rb +1 -1
  214. data/lib/karafka.rb +36 -36
  215. metadata +7 -3
@@ -60,14 +60,14 @@ module Karafka
60
60
  # Validate consumer group settings
61
61
  Contracts::ConsumerGroup.new.validate!(
62
62
  consumer_group.to_h,
63
- scope: ['routes', consumer_group.name]
63
+ scope: ["routes", consumer_group.name]
64
64
  )
65
65
 
66
66
  # and then its topics settings
67
67
  consumer_group.topics.each do |topic|
68
68
  Contracts::Topic.new.validate!(
69
69
  topic.to_h,
70
- scope: ['routes', consumer_group.name, topic.name]
70
+ scope: ["routes", consumer_group.name, topic.name]
71
71
  )
72
72
  end
73
73
 
@@ -78,7 +78,7 @@ module Karafka
78
78
  end
79
79
 
80
80
  # Clear out the drawn routes.
81
- alias array_clear clear
81
+ alias_method :array_clear, :clear
82
82
  private :array_clear
83
83
 
84
84
  # Clear routes and draw them again with the given block. Helpful for testing purposes.
@@ -8,8 +8,8 @@ module Karafka
8
8
  class ConsumerGroup < Karafka::Contracts::Base
9
9
  configure do |config|
10
10
  config.error_messages = YAML.safe_load_file(
11
- File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
12
- ).fetch('en').fetch('validations').fetch('routing').fetch('consumer_group')
11
+ File.join(Karafka.gem_root, "config", "locales", "errors.yml")
12
+ ).fetch("en").fetch("validations").fetch("routing").fetch("consumer_group")
13
13
  end
14
14
 
15
15
  required(:id) { |val| val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val) }
@@ -49,18 +49,18 @@ module Karafka
49
49
  names = data.fetch(:topics).map { |topic| topic[:name] }
50
50
  names_hash = names.each_with_object({}) { |n, h| h[n] = true }
51
51
  error_occured = false
52
- namespace_chars = ['.', '_'].freeze
52
+ namespace_chars = [".", "_"].freeze
53
53
  names.each do |n|
54
54
  # Skip topic names that are not namespaced
55
55
  next unless n.chars.find { |c| namespace_chars.include?(c) }
56
56
 
57
- if n.chars.include?('.')
57
+ if n.chars.include?(".")
58
58
  # Check underscore styled topic
59
- underscored_topic = n.tr('.', '_')
59
+ underscored_topic = n.tr(".", "_")
60
60
  error_occured = names_hash[underscored_topic] ? true : false
61
61
  else
62
62
  # Check dot styled topic
63
- dot_topic = n.tr('_', '.')
63
+ dot_topic = n.tr("_", ".")
64
64
  error_occured = names_hash[dot_topic] ? true : false
65
65
  end
66
66
  end
@@ -7,8 +7,8 @@ module Karafka
7
7
  class Routing < Karafka::Contracts::Base
8
8
  configure do |config|
9
9
  config.error_messages = YAML.safe_load_file(
10
- File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
11
- ).fetch('en').fetch('validations').fetch('routing')
10
+ File.join(Karafka.gem_root, "config", "locales", "errors.yml")
11
+ ).fetch("en").fetch("validations").fetch("routing")
12
12
  end
13
13
 
14
14
  # Ensures, that when declarative topics strict requirement is on, all topics have
@@ -7,8 +7,8 @@ module Karafka
7
7
  class Topic < Karafka::Contracts::Base
8
8
  configure do |config|
9
9
  config.error_messages = YAML.safe_load_file(
10
- File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
11
- ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
10
+ File.join(Karafka.gem_root, "config", "locales", "errors.yml")
11
+ ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
12
12
  end
13
13
 
14
14
  required(:deserializers) { |val| !val.nil? }
@@ -59,7 +59,7 @@ module Karafka
59
59
 
60
60
  kafka = data.fetch(:kafka)
61
61
 
62
- next if kafka.key?(:'bootstrap.servers')
62
+ next if kafka.key?(:"bootstrap.servers")
63
63
 
64
64
  [[%w[kafka bootstrap.servers], :missing]]
65
65
  end
@@ -69,7 +69,7 @@ module Karafka
69
69
  next unless Karafka::App.config.strict_topics_namespacing
70
70
 
71
71
  value = data.fetch(:name)
72
- namespace_chars = ['.', '_'].freeze
72
+ namespace_chars = [".", "_"].freeze
73
73
  namespacing_chars_count = value.chars.find_all do |c|
74
74
  namespace_chars.include?(c)
75
75
  end.uniq.size
@@ -11,15 +11,15 @@ module Karafka
11
11
  class Topic < Karafka::Contracts::Base
12
12
  configure do |config|
13
13
  config.error_messages = YAML.safe_load_file(
14
- File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
15
- ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
14
+ File.join(Karafka.gem_root, "config", "locales", "errors.yml")
15
+ ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
16
16
  end
17
17
 
18
18
  virtual do |data, errors|
19
19
  next unless errors.empty?
20
20
  next unless data[:active_job][:active]
21
21
  # One should not define active job jobs without ActiveJob being available for usage
22
- next if Object.const_defined?('ActiveJob::Base')
22
+ next if Object.const_defined?("ActiveJob::Base")
23
23
 
24
24
  [[%i[consumer], :active_job_missing]]
25
25
  end
@@ -37,19 +37,19 @@ module Karafka
37
37
  result = super(&block)
38
38
 
39
39
  each do |consumer_group|
40
- if scope::Contracts.const_defined?('ConsumerGroup', false)
40
+ if scope::Contracts.const_defined?("ConsumerGroup", false)
41
41
  scope::Contracts::ConsumerGroup.new.validate!(
42
42
  consumer_group.to_h,
43
- scope: ['routes', consumer_group.name]
43
+ scope: ["routes", consumer_group.name]
44
44
  )
45
45
  end
46
46
 
47
- next unless scope::Contracts.const_defined?('Topic', false)
47
+ next unless scope::Contracts.const_defined?("Topic", false)
48
48
 
49
49
  consumer_group.topics.each do |topic|
50
50
  scope::Contracts::Topic.new.validate!(
51
51
  topic.to_h,
52
- scope: ['routes', consumer_group.name, topic.name]
52
+ scope: ["routes", consumer_group.name, topic.name]
53
53
  )
54
54
  end
55
55
  end
@@ -13,35 +13,35 @@ module Karafka
13
13
  class << self
14
14
  # Extends topic and builder with given feature API
15
15
  def activate
16
- if const_defined?('Topic', false)
16
+ if const_defined?("Topic", false)
17
17
  Topic.prepend(self::Topic)
18
18
  end
19
19
 
20
- if const_defined?('Topics', false)
20
+ if const_defined?("Topics", false)
21
21
  Topics.prepend(self::Topics)
22
22
  end
23
23
 
24
- if const_defined?('ConsumerGroup', false)
24
+ if const_defined?("ConsumerGroup", false)
25
25
  ConsumerGroup.prepend(self::ConsumerGroup)
26
26
  end
27
27
 
28
- if const_defined?('Proxy', false)
28
+ if const_defined?("Proxy", false)
29
29
  Proxy.prepend(self::Proxy)
30
30
  end
31
31
 
32
- if const_defined?('Builder', false)
32
+ if const_defined?("Builder", false)
33
33
  Builder.prepend(self::Builder)
34
34
  end
35
35
 
36
- if const_defined?('Contracts', false)
36
+ if const_defined?("Contracts", false)
37
37
  Builder.prepend(Base::Expander.new(self))
38
38
  end
39
39
 
40
- if const_defined?('SubscriptionGroup', false)
40
+ if const_defined?("SubscriptionGroup", false)
41
41
  SubscriptionGroup.prepend(self::SubscriptionGroup)
42
42
  end
43
43
 
44
- if const_defined?('SubscriptionGroupsBuilder', false)
44
+ if const_defined?("SubscriptionGroupsBuilder", false)
45
45
  SubscriptionGroupsBuilder.prepend(self::SubscriptionGroupsBuilder)
46
46
  end
47
47
  end
@@ -10,8 +10,8 @@ module Karafka
10
10
  class Topic < Karafka::Contracts::Base
11
11
  configure do |config|
12
12
  config.error_messages = YAML.safe_load_file(
13
- File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
14
- ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
13
+ File.join(Karafka.gem_root, "config", "locales", "errors.yml")
14
+ ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
15
15
  end
16
16
 
17
17
  nested :dead_letter_queue do
@@ -10,8 +10,8 @@ module Karafka
10
10
  class Topic < Karafka::Contracts::Base
11
11
  configure do |config|
12
12
  config.error_messages = YAML.safe_load_file(
13
- File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
14
- ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
13
+ File.join(Karafka.gem_root, "config", "locales", "errors.yml")
14
+ ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
15
15
  end
16
16
 
17
17
  nested :declaratives do
@@ -10,8 +10,8 @@ module Karafka
10
10
  class Topic < Karafka::Contracts::Base
11
11
  configure do |config|
12
12
  config.error_messages = YAML.safe_load_file(
13
- File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
14
- ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
13
+ File.join(Karafka.gem_root, "config", "locales", "errors.yml")
14
+ ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
15
15
  end
16
16
 
17
17
  nested :deserializers do
@@ -10,8 +10,8 @@ module Karafka
10
10
  class Topic < Karafka::Contracts::Base
11
11
  configure do |config|
12
12
  config.error_messages = YAML.safe_load_file(
13
- File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
14
- ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
13
+ File.join(Karafka.gem_root, "config", "locales", "errors.yml")
14
+ ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
15
15
  end
16
16
 
17
17
  nested :eofed do
@@ -25,7 +25,7 @@ module Karafka
25
25
 
26
26
  next unless eofed[:active]
27
27
 
28
- next if data[:kafka][:'enable.partition.eof']
28
+ next if data[:kafka][:"enable.partition.eof"]
29
29
 
30
30
  [[%i[eofed kafka], :enable]]
31
31
  end
@@ -10,8 +10,8 @@ module Karafka
10
10
  class Topic < Karafka::Contracts::Base
11
11
  configure do |config|
12
12
  config.error_messages = YAML.safe_load_file(
13
- File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
14
- ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
13
+ File.join(Karafka.gem_root, "config", "locales", "errors.yml")
14
+ ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
15
15
  end
16
16
 
17
17
  nested :inline_insights do
@@ -12,17 +12,17 @@ module Karafka
12
12
  #
13
13
  # @param _config [Karafka::Core::Configurable::Node] app config
14
14
  def post_setup(_config)
15
- Karafka::App.monitor.subscribe('app.running') do
15
+ Karafka::App.monitor.subscribe("app.running") do
16
16
  # Do not activate tracking of statistics if none of our active topics uses it
17
17
  # This prevents us from tracking metrics when user just runs a subset of topics
18
18
  # in a given process and none of those actually utilizes this feature
19
19
  next unless Karafka::App
20
- .subscription_groups
21
- .values
22
- .flat_map(&:itself)
23
- .flat_map(&:topics)
24
- .flat_map(&:to_a)
25
- .any?(&:inline_insights?)
20
+ .subscription_groups
21
+ .values
22
+ .flat_map(&:itself)
23
+ .flat_map(&:topics)
24
+ .flat_map(&:to_a)
25
+ .any?(&:inline_insights?)
26
26
 
27
27
  # Initialize the tracker prior to becoming multi-threaded
28
28
  Karafka::Processing::InlineInsights::Tracker.instance
@@ -10,8 +10,8 @@ module Karafka
10
10
  class Topic < Karafka::Contracts::Base
11
11
  configure do |config|
12
12
  config.error_messages = YAML.safe_load_file(
13
- File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
14
- ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
13
+ File.join(Karafka.gem_root, "config", "locales", "errors.yml")
14
+ ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
15
15
  end
16
16
 
17
17
  nested :manual_offset_management do
@@ -58,7 +58,7 @@ module Karafka
58
58
 
59
59
  # @return [String] consumer group id
60
60
  def consumer_group_id
61
- kafka[:'group.id']
61
+ kafka[:"group.id"]
62
62
  end
63
63
 
64
64
  # @return [Integer] max messages fetched in a single go
@@ -107,7 +107,7 @@ module Karafka
107
107
  # node identifier. This refreshes this if needed when in swarm.
108
108
  def refresh
109
109
  return unless node
110
- return unless kafka.key?(:'group.instance.id')
110
+ return unless kafka.key?(:"group.instance.id")
111
111
 
112
112
  @kafka = build_kafka
113
113
  end
@@ -124,11 +124,11 @@ module Karafka
124
124
  inject_group_instance_id(kafka)
125
125
  inject_client_id(kafka)
126
126
 
127
- kafka[:'group.id'] ||= @consumer_group.id
128
- kafka[:'auto.offset.reset'] ||= @topics.first.initial_offset
127
+ kafka[:"group.id"] ||= @consumer_group.id
128
+ kafka[:"auto.offset.reset"] ||= @topics.first.initial_offset
129
129
  # Karafka manages the offsets based on the processing state, thus we do not rely on the
130
130
  # rdkafka offset auto-storing
131
- kafka[:'enable.auto.offset.store'] = false
131
+ kafka[:"enable.auto.offset.store"] = false
132
132
  kafka.freeze
133
133
  kafka
134
134
  end
@@ -146,9 +146,9 @@ module Karafka
146
146
  def inject_client_id(kafka)
147
147
  # If client id is set directly on librdkafka level, we do nothing and just go with what
148
148
  # end user has configured
149
- return if kafka.key?(:'client.id')
149
+ return if kafka.key?(:"client.id")
150
150
 
151
- kafka[:'client.id'] = client_id
151
+ kafka[:"client.id"] = client_id
152
152
  end
153
153
 
154
154
  # If we use static group memberships, there can be a case, where same instance id would
@@ -161,7 +161,7 @@ module Karafka
161
161
  # affecting the instance id and causing conflicts
162
162
  # @param kafka [Hash] kafka level config
163
163
  def inject_group_instance_id(kafka)
164
- group_instance_prefix = kafka.fetch(:'group.instance.id', false)
164
+ group_instance_prefix = kafka.fetch(:"group.instance.id", false)
165
165
 
166
166
  # If group instance id was not even configured, do nothing
167
167
  return unless group_instance_prefix
@@ -170,7 +170,7 @@ module Karafka
170
170
  # have different instances ids but they are reproducible
171
171
  components = [group_instance_prefix, node ? node.id : nil, @position]
172
172
 
173
- kafka[:'group.instance.id'] = components.compact.join('_')
173
+ kafka[:"group.instance.id"] = components.compact.join("_")
174
174
  end
175
175
  end
176
176
  end
@@ -59,12 +59,12 @@ module Karafka
59
59
  workers.each(&:join)
60
60
  # If anything crashes here, we need to raise the error and crush the runner because it means
61
61
  # that something terrible happened
62
- rescue StandardError => e
62
+ rescue => e
63
63
  Karafka.monitor.instrument(
64
- 'error.occurred',
64
+ "error.occurred",
65
65
  caller: self,
66
66
  error: e,
67
- type: 'runner.call.error'
67
+ type: "runner.call.error"
68
68
  )
69
69
  Karafka::App.stop!
70
70
  raise e
@@ -126,11 +126,20 @@ module Karafka
126
126
 
127
127
  raise Errors::ForcefulShutdownError
128
128
  rescue Errors::ForcefulShutdownError => e
129
+ active_listeners = listeners.select(&:active?)
130
+ alive_workers = workers.select(&:alive?)
131
+
132
+ # Collect details about subscription groups that still have jobs in processing
133
+ in_processing = jobs_queue ? jobs_queue.in_processing : {}
134
+
129
135
  Karafka.monitor.instrument(
130
- 'error.occurred',
136
+ "error.occurred",
131
137
  caller: self,
132
138
  error: e,
133
- type: 'app.stopping.error'
139
+ active_listeners: active_listeners,
140
+ alive_workers: alive_workers,
141
+ in_processing: in_processing,
142
+ type: "app.stopping.error"
134
143
  )
135
144
 
136
145
  # We're done waiting, lets kill them!
@@ -144,13 +153,13 @@ module Karafka
144
153
  # indefinitely even with risk of VM crash as this is a last resort.
145
154
  Thread.new do
146
155
  listeners.each(&:shutdown)
147
- rescue StandardError => e
156
+ rescue => e
148
157
  # If anything wrong happened during shutdown, we also want to record it
149
158
  Karafka.monitor.instrument(
150
- 'error.occurred',
159
+ "error.occurred",
151
160
  caller: self,
152
161
  error: e,
153
- type: 'app.forceful_stopping.error'
162
+ type: "app.forceful_stopping.error"
154
163
  )
155
164
  end.join(forceful_shutdown_wait / 1_000.0)
156
165
 
@@ -334,7 +334,7 @@ module Karafka
334
334
  ].freeze
335
335
 
336
336
  # Location of the file with rdkafka settings list
337
- SOURCE = <<~SOURCE.delete("\n").gsub(/\s+/, '/')
337
+ SOURCE = <<~SOURCE.delete("\n").gsub(/\s+/, "/")
338
338
  https://raw.githubusercontent.com
339
339
  confluentinc/librdkafka
340
340
  v#{Rdkafka::LIBRDKAFKA_VERSION}
@@ -365,21 +365,21 @@ module Karafka
365
365
  # options list in case it would change
366
366
  def generate
367
367
  # Not used anywhere else, hence required here
368
- require 'open-uri'
368
+ require "open-uri"
369
369
 
370
370
  attributes = { consumer: Set.new, producer: Set.new }
371
371
 
372
372
  URI.parse(SOURCE).open.readlines.each do |line|
373
- next unless line.include?('|')
373
+ next unless line.include?("|")
374
374
 
375
- attribute, attribute_type = line.split('|').map(&:strip)
375
+ attribute, attribute_type = line.split("|").map(&:strip)
376
376
 
377
377
  case attribute_type
378
- when 'C'
378
+ when "C"
379
379
  attributes[:consumer] << attribute
380
- when 'P'
380
+ when "P"
381
381
  attributes[:producer] << attribute
382
- when '*'
382
+ when "*"
383
383
  attributes[:consumer] << attribute
384
384
  attributes[:producer] << attribute
385
385
  else
@@ -26,12 +26,12 @@ module Karafka
26
26
  # terms and conditions
27
27
  setting :token, default: false
28
28
  # option entity [String] for whom we did issue the license
29
- setting :entity, default: ''
29
+ setting :entity, default: ""
30
30
  end
31
31
 
32
32
  # option client_id [String] kafka client_id - used to uniquely identify given client instance
33
33
  # Used only for logging.
34
- setting :client_id, default: 'karafka'
34
+ setting :client_id, default: "karafka"
35
35
  # option logger [Instance] logger that we want to use
36
36
  setting :logger, default: Karafka::Instrumentation::Logger.new
37
37
  # option monitor [Instance] monitor that we will to use (defaults to Karafka::Monitor)
@@ -41,7 +41,7 @@ module Karafka
41
41
  setting :consumer_persistence, default: true
42
42
  # option [String] should we start with the earliest possible offset or latest
43
43
  # This will set the `auto.offset.reset` value unless present in the kafka scope
44
- setting :initial_offset, default: 'earliest'
44
+ setting :initial_offset, default: "earliest"
45
45
  # options max_messages [Integer] how many messages do we want to fetch from Kafka in one go
46
46
  setting :max_messages, default: 100
47
47
  # option [Integer] number of milliseconds we can wait while fetching data
@@ -71,7 +71,7 @@ module Karafka
71
71
  # and/or we work with existing systems where we cannot change topics names.
72
72
  setting :strict_topics_namespacing, default: true
73
73
  # option [String] default consumer group name for implicit routing
74
- setting :group_id, default: 'app'
74
+ setting :group_id, default: "app"
75
75
  # option [Boolean] when set to true, it will validate as part of the routing validation, that
76
76
  # all topics and DLQ topics (even not active) have the declarative topics definitions.
77
77
  # Really useful when you want to ensure that all topics in routing are managed via
@@ -115,22 +115,22 @@ module Karafka
115
115
  # option [Hash] extra changes to the default root kafka settings
116
116
  setting :kafka, default: {
117
117
  # We want to know when there is no more data not to end up with an endless loop
118
- 'enable.partition.eof': true,
118
+ "enable.partition.eof": true,
119
119
  # Do not publish statistics from admin as they are not relevant
120
- 'statistics.interval.ms': 0,
120
+ "statistics.interval.ms": 0,
121
121
  # Fetch at most 5 MBs when using admin
122
- 'fetch.message.max.bytes': 5 * 1_048_576,
122
+ "fetch.message.max.bytes": 5 * 1_048_576,
123
123
  # Do not commit offset automatically, this prevents offset tracking for operations
124
124
  # involving a consumer instance
125
- 'enable.auto.commit': false,
125
+ "enable.auto.commit": false,
126
126
  # Make sure that topic metadata lookups do not create topics accidentally
127
- 'allow.auto.create.topics': false,
127
+ "allow.auto.create.topics": false,
128
128
  # Do not store offsets automatically in admin in any way
129
- 'enable.auto.offset.store': false
129
+ "enable.auto.offset.store": false
130
130
  }
131
131
 
132
132
  # option [String] default name for the admin consumer group.
133
- setting :group_id, default: 'karafka_admin'
133
+ setting :group_id, default: "karafka_admin"
134
134
 
135
135
  # option max_wait_time [Integer] We wait only for this amount of time before raising error
136
136
  # as we intercept this error and retry after checking that the operation was finished or
@@ -13,8 +13,8 @@ module Karafka
13
13
  class Config < Karafka::Contracts::Base
14
14
  configure do |config|
15
15
  config.error_messages = YAML.safe_load_file(
16
- File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
17
- ).fetch('en').fetch('validations').fetch('setup').fetch('config')
16
+ File.join(Karafka.gem_root, "config", "locales", "errors.yml")
17
+ ).fetch("en").fetch("validations").fetch("setup").fetch("config")
18
18
  end
19
19
 
20
20
  # Topics regexp constant reference for easier usage
@@ -10,18 +10,18 @@ module Karafka
10
10
  CONSUMER_KAFKA_DEFAULTS = {
11
11
  # We emit the statistics by default, so all the instrumentation and web-ui work out of
12
12
  # the box, without requiring users to take any extra actions aside from enabling.
13
- 'statistics.interval.ms': 5_000,
14
- 'client.software.name': 'karafka',
13
+ "statistics.interval.ms": 5_000,
14
+ "client.software.name": "karafka",
15
15
  # Same as librdkafka default, we inject it nonetheless to have it always available as
16
16
  # some features may use this value for computation and it is better to ensure, we do
17
17
  # always have it
18
- 'max.poll.interval.ms': 300_000,
19
- 'socket.nagle.disable': true,
20
- 'client.software.version': [
18
+ "max.poll.interval.ms": 300_000,
19
+ "socket.nagle.disable": true,
20
+ "client.software.version": [
21
21
  "v#{Karafka::VERSION}",
22
22
  "rdkafka-ruby-v#{Rdkafka::VERSION}",
23
23
  "librdkafka-v#{Rdkafka::LIBRDKAFKA_VERSION}"
24
- ].join('-')
24
+ ].join("-")
25
25
  }.freeze
26
26
 
27
27
  # Contains settings that should not be used in production but make life easier in dev
@@ -29,12 +29,12 @@ module Karafka
29
29
  # Will create non-existing topics automatically.
30
30
  # Note that the broker needs to be configured with `auto.create.topics.enable=true`
31
31
  # While it is not recommended in prod, it simplifies work in dev
32
- 'allow.auto.create.topics': 'true',
32
+ "allow.auto.create.topics": "true",
33
33
  # We refresh the cluster state often as newly created topics in dev may not be detected
34
34
  # fast enough. Fast enough means within reasonable time to provide decent user experience
35
35
  # While it's only a one time thing for new topics, it can still be irritating to have to
36
36
  # restart the process.
37
- 'topic.metadata.refresh.interval.ms': 5_000
37
+ "topic.metadata.refresh.interval.ms": 5_000
38
38
  }.freeze
39
39
 
40
40
  # Contains settings that should not be used in production but make life easier in dev
@@ -42,9 +42,9 @@ module Karafka
42
42
  # they have to set this by themselves.
43
43
  PRODUCER_KAFKA_DEV_DEFAULTS = {
44
44
  # For all of those same reasoning as for the consumer
45
- 'allow.auto.create.topics': 'true',
46
- 'topic.metadata.refresh.interval.ms': 5_000,
47
- 'socket.nagle.disable': true
45
+ "allow.auto.create.topics": "true",
46
+ "topic.metadata.refresh.interval.ms": 5_000,
47
+ "socket.nagle.disable": true
48
48
  }.freeze
49
49
 
50
50
  private_constant(
@@ -77,7 +77,7 @@ module Karafka
77
77
 
78
78
  # Checks on nodes if they are ok one after another
79
79
  def control
80
- monitor.instrument('swarm.manager.control', caller: self) do
80
+ monitor.instrument("swarm.manager.control", caller: self) do
81
81
  @nodes.each do |node|
82
82
  statuses = @statuses[node]
83
83
 
@@ -107,7 +107,7 @@ module Karafka
107
107
  # Do not run any other checks on this node if it is during stopping but still has time
108
108
  return true unless over?(statuses[:stop], shutdown_timeout)
109
109
 
110
- monitor.instrument('swarm.manager.terminating', caller: self, node: node) do
110
+ monitor.instrument("swarm.manager.terminating", caller: self, node: node) do
111
111
  node.terminate
112
112
  statuses[:terminate] = monotonic_now
113
113
  end
@@ -135,7 +135,7 @@ module Karafka
135
135
  else
136
136
  # A single invalid report will cause it to stop. We do not support intermediate failures
137
137
  # that would recover. Such states should be implemented in the listener.
138
- monitor.instrument('swarm.manager.stopping', caller: self, node: node, status: status) do
138
+ monitor.instrument("swarm.manager.stopping", caller: self, node: node, status: status) do
139
139
  node.stop
140
140
  statuses[:stop] = monotonic_now
141
141
  end
@@ -157,7 +157,7 @@ module Karafka
157
157
 
158
158
  # Start the stopping procedure if the node stopped reporting frequently enough
159
159
  monitor.instrument(
160
- 'swarm.manager.stopping',
160
+ "swarm.manager.stopping",
161
161
  caller: self,
162
162
  node: node,
163
163
  status: NOT_RESPONDING_SHUTDOWN_STATUS
@@ -210,9 +210,9 @@ module Karafka
210
210
  statuses.clear
211
211
  statuses[:control] = monotonic_now
212
212
 
213
- monitor.instrument('swarm.manager.before_fork', instr_args)
213
+ monitor.instrument("swarm.manager.before_fork", instr_args)
214
214
  node.start
215
- monitor.instrument('swarm.manager.after_fork', instr_args)
215
+ monitor.instrument("swarm.manager.after_fork", instr_args)
216
216
 
217
217
  node
218
218
  end