karafka 2.5.5 → 2.5.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (215) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +20 -0
  3. data/LICENSE-COMM +4 -0
  4. data/README.md +2 -2
  5. data/certs/expired.txt +2 -0
  6. data/karafka.gemspec +23 -23
  7. data/lib/active_job/karafka.rb +2 -2
  8. data/lib/active_job/queue_adapters/karafka_adapter.rb +5 -5
  9. data/lib/karafka/active_job/consumer.rb +3 -3
  10. data/lib/karafka/active_job/current_attributes.rb +4 -4
  11. data/lib/karafka/active_job/job_options_contract.rb +2 -2
  12. data/lib/karafka/admin/acl.rb +3 -3
  13. data/lib/karafka/admin/configs/resource.rb +1 -1
  14. data/lib/karafka/admin/configs.rb +1 -1
  15. data/lib/karafka/admin/consumer_groups.rb +8 -8
  16. data/lib/karafka/admin/contracts/replication.rb +2 -2
  17. data/lib/karafka/admin/replication.rb +21 -21
  18. data/lib/karafka/admin/topics.rb +6 -6
  19. data/lib/karafka/admin.rb +4 -5
  20. data/lib/karafka/app.rb +3 -3
  21. data/lib/karafka/base_consumer.rb +34 -30
  22. data/lib/karafka/cli/base.rb +8 -8
  23. data/lib/karafka/cli/console.rb +1 -1
  24. data/lib/karafka/cli/contracts/server.rb +12 -12
  25. data/lib/karafka/cli/help.rb +2 -2
  26. data/lib/karafka/cli/info.rb +4 -4
  27. data/lib/karafka/cli/install.rb +11 -11
  28. data/lib/karafka/cli/server.rb +6 -6
  29. data/lib/karafka/cli/swarm.rb +1 -1
  30. data/lib/karafka/cli/topics/align.rb +4 -4
  31. data/lib/karafka/cli/topics/base.rb +5 -5
  32. data/lib/karafka/cli/topics/create.rb +2 -2
  33. data/lib/karafka/cli/topics/delete.rb +2 -2
  34. data/lib/karafka/cli/topics/help.rb +5 -1
  35. data/lib/karafka/cli/topics/plan.rb +16 -16
  36. data/lib/karafka/cli/topics/repartition.rb +3 -3
  37. data/lib/karafka/cli/topics.rb +22 -22
  38. data/lib/karafka/cli.rb +2 -2
  39. data/lib/karafka/connection/client.rb +17 -17
  40. data/lib/karafka/connection/listener.rb +6 -6
  41. data/lib/karafka/connection/mode.rb +1 -1
  42. data/lib/karafka/connection/proxy.rb +1 -1
  43. data/lib/karafka/connection/status.rb +2 -2
  44. data/lib/karafka/constraints.rb +3 -3
  45. data/lib/karafka/embedded.rb +3 -3
  46. data/lib/karafka/env.rb +4 -4
  47. data/lib/karafka/errors.rb +6 -1
  48. data/lib/karafka/execution_mode.rb +1 -1
  49. data/lib/karafka/helpers/config_importer.rb +2 -2
  50. data/lib/karafka/helpers/interval_runner.rb +4 -2
  51. data/lib/karafka/helpers/multi_delegator.rb +1 -1
  52. data/lib/karafka/instrumentation/assignments_tracker.rb +9 -9
  53. data/lib/karafka/instrumentation/callbacks/error.rb +5 -5
  54. data/lib/karafka/instrumentation/callbacks/oauthbearer_token_refresh.rb +4 -4
  55. data/lib/karafka/instrumentation/callbacks/rebalance.rb +6 -6
  56. data/lib/karafka/instrumentation/callbacks/statistics.rb +5 -5
  57. data/lib/karafka/instrumentation/logger.rb +7 -7
  58. data/lib/karafka/instrumentation/logger_listener.rb +76 -63
  59. data/lib/karafka/instrumentation/vendors/appsignal/base.rb +1 -1
  60. data/lib/karafka/instrumentation/vendors/appsignal/client.rb +1 -1
  61. data/lib/karafka/instrumentation/vendors/appsignal/errors_listener.rb +1 -1
  62. data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb +36 -36
  63. data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +33 -28
  64. data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +38 -38
  65. data/lib/karafka/instrumentation/vendors/kubernetes/base_listener.rb +5 -5
  66. data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +1 -1
  67. data/lib/karafka/instrumentation/vendors/kubernetes/swarm_liveness_listener.rb +1 -1
  68. data/lib/karafka/licenser.rb +115 -8
  69. data/lib/karafka/messages/builders/batch_metadata.rb +4 -2
  70. data/lib/karafka/messages/messages.rb +1 -1
  71. data/lib/karafka/patches/rdkafka/bindings.rb +2 -2
  72. data/lib/karafka/pro/active_job/job_options_contract.rb +2 -2
  73. data/lib/karafka/pro/cleaner/messages/messages.rb +10 -0
  74. data/lib/karafka/pro/cli/contracts/server.rb +12 -12
  75. data/lib/karafka/pro/cli/parallel_segments/base.rb +4 -4
  76. data/lib/karafka/pro/cli/parallel_segments/collapse.rb +5 -5
  77. data/lib/karafka/pro/cli/parallel_segments/distribute.rb +3 -3
  78. data/lib/karafka/pro/cli/parallel_segments.rb +7 -7
  79. data/lib/karafka/pro/cli/topics/health.rb +162 -0
  80. data/lib/karafka/pro/cli/topics.rb +52 -0
  81. data/lib/karafka/pro/connection/manager.rb +14 -14
  82. data/lib/karafka/pro/encryption/contracts/config.rb +2 -2
  83. data/lib/karafka/pro/encryption/messages/middleware.rb +2 -2
  84. data/lib/karafka/pro/encryption/messages/parser.rb +2 -2
  85. data/lib/karafka/pro/encryption/setup/config.rb +2 -2
  86. data/lib/karafka/pro/iterator/tpl_builder.rb +2 -2
  87. data/lib/karafka/pro/iterator.rb +1 -1
  88. data/lib/karafka/pro/loader.rb +2 -1
  89. data/lib/karafka/pro/processing/adaptive_iterator/consumer.rb +1 -1
  90. data/lib/karafka/pro/processing/coordinators/virtual_offset_manager.rb +24 -14
  91. data/lib/karafka/pro/processing/filters/base.rb +1 -1
  92. data/lib/karafka/pro/processing/filters/delayer.rb +2 -2
  93. data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb +1 -1
  94. data/lib/karafka/pro/processing/offset_metadata/consumer.rb +1 -1
  95. data/lib/karafka/pro/processing/parallel_segments/filters/base.rb +6 -6
  96. data/lib/karafka/pro/processing/partitioner.rb +3 -3
  97. data/lib/karafka/pro/processing/periodic_job/consumer.rb +6 -5
  98. data/lib/karafka/pro/processing/piping/consumer.rb +7 -7
  99. data/lib/karafka/pro/processing/schedulers/base.rb +5 -5
  100. data/lib/karafka/pro/processing/schedulers/default.rb +5 -5
  101. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb +6 -3
  102. data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb +6 -3
  103. data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb +6 -3
  104. data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb +2 -2
  105. data/lib/karafka/pro/processing/strategies/default.rb +22 -22
  106. data/lib/karafka/pro/processing/strategies/dlq/default.rb +7 -7
  107. data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb +6 -3
  108. data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb +6 -3
  109. data/lib/karafka/pro/processing/strategies/ftr/default.rb +2 -2
  110. data/lib/karafka/pro/processing/strategies/lrj/default.rb +2 -2
  111. data/lib/karafka/pro/processing/strategies/lrj/ftr.rb +6 -3
  112. data/lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb +6 -3
  113. data/lib/karafka/pro/processing/strategies/lrj/mom.rb +2 -2
  114. data/lib/karafka/pro/recurring_tasks/consumer.rb +2 -2
  115. data/lib/karafka/pro/recurring_tasks/contracts/config.rb +2 -2
  116. data/lib/karafka/pro/recurring_tasks/contracts/task.rb +2 -2
  117. data/lib/karafka/pro/recurring_tasks/dispatcher.rb +2 -2
  118. data/lib/karafka/pro/recurring_tasks/listener.rb +1 -1
  119. data/lib/karafka/pro/recurring_tasks/matcher.rb +2 -2
  120. data/lib/karafka/pro/recurring_tasks/serializer.rb +5 -5
  121. data/lib/karafka/pro/recurring_tasks/setup/config.rb +3 -3
  122. data/lib/karafka/pro/recurring_tasks/task.rb +4 -4
  123. data/lib/karafka/pro/recurring_tasks.rb +4 -4
  124. data/lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb +2 -2
  125. data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +2 -2
  126. data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb +1 -1
  127. data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb +2 -2
  128. data/lib/karafka/pro/routing/features/direct_assignments/contracts/consumer_group.rb +2 -2
  129. data/lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb +2 -2
  130. data/lib/karafka/pro/routing/features/direct_assignments/topic.rb +1 -1
  131. data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb +2 -2
  132. data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb +2 -2
  133. data/lib/karafka/pro/routing/features/inline_insights/contracts/topic.rb +2 -2
  134. data/lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb +2 -2
  135. data/lib/karafka/pro/routing/features/long_running_job/topic.rb +1 -1
  136. data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb +2 -2
  137. data/lib/karafka/pro/routing/features/multiplexing.rb +5 -5
  138. data/lib/karafka/pro/routing/features/non_blocking_job/topic.rb +1 -1
  139. data/lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb +2 -2
  140. data/lib/karafka/pro/routing/features/offset_metadata/topic.rb +1 -1
  141. data/lib/karafka/pro/routing/features/offset_metadata.rb +1 -1
  142. data/lib/karafka/pro/routing/features/parallel_segments/consumer_group.rb +5 -5
  143. data/lib/karafka/pro/routing/features/parallel_segments/contracts/consumer_group.rb +2 -2
  144. data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +2 -2
  145. data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb +3 -3
  146. data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +2 -2
  147. data/lib/karafka/pro/routing/features/patterns/topic.rb +1 -1
  148. data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb +2 -2
  149. data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb +2 -2
  150. data/lib/karafka/pro/routing/features/periodic_job/topic.rb +1 -1
  151. data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb +7 -7
  152. data/lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb +2 -2
  153. data/lib/karafka/pro/routing/features/scheduled_messages/builder.rb +13 -13
  154. data/lib/karafka/pro/routing/features/scheduled_messages/contracts/topic.rb +2 -2
  155. data/lib/karafka/pro/routing/features/swarm/contracts/routing.rb +2 -2
  156. data/lib/karafka/pro/routing/features/swarm/contracts/topic.rb +2 -2
  157. data/lib/karafka/pro/routing/features/swarm.rb +1 -1
  158. data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb +2 -2
  159. data/lib/karafka/pro/routing/features/virtual_partitions/config.rb +7 -7
  160. data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +2 -2
  161. data/lib/karafka/pro/scheduled_messages/consumer.rb +4 -4
  162. data/lib/karafka/pro/scheduled_messages/contracts/config.rb +2 -2
  163. data/lib/karafka/pro/scheduled_messages/contracts/message.rb +10 -10
  164. data/lib/karafka/pro/scheduled_messages/daily_buffer.rb +2 -2
  165. data/lib/karafka/pro/scheduled_messages/deserializers/headers.rb +4 -4
  166. data/lib/karafka/pro/scheduled_messages/dispatcher.rb +5 -5
  167. data/lib/karafka/pro/scheduled_messages/proxy.rb +8 -8
  168. data/lib/karafka/pro/scheduled_messages/schema_validator.rb +1 -1
  169. data/lib/karafka/pro/scheduled_messages/setup/config.rb +2 -2
  170. data/lib/karafka/pro/scheduled_messages/state.rb +1 -1
  171. data/lib/karafka/pro/scheduled_messages/tracker.rb +2 -2
  172. data/lib/karafka/pro/scheduled_messages.rb +2 -2
  173. data/lib/karafka/pro/swarm/liveness_listener.rb +2 -2
  174. data/lib/karafka/process.rb +1 -1
  175. data/lib/karafka/processing/coordinator.rb +1 -1
  176. data/lib/karafka/processing/inline_insights/consumer.rb +4 -4
  177. data/lib/karafka/processing/inline_insights/tracker.rb +6 -6
  178. data/lib/karafka/processing/jobs/base.rb +6 -4
  179. data/lib/karafka/processing/jobs_queue.rb +10 -0
  180. data/lib/karafka/processing/schedulers/default.rb +4 -4
  181. data/lib/karafka/processing/strategies/base.rb +6 -6
  182. data/lib/karafka/processing/strategies/default.rb +13 -13
  183. data/lib/karafka/processing/strategies/dlq.rb +1 -1
  184. data/lib/karafka/processing/worker.rb +5 -5
  185. data/lib/karafka/railtie.rb +11 -11
  186. data/lib/karafka/routing/builder.rb +3 -3
  187. data/lib/karafka/routing/contracts/consumer_group.rb +6 -6
  188. data/lib/karafka/routing/contracts/routing.rb +2 -2
  189. data/lib/karafka/routing/contracts/topic.rb +4 -4
  190. data/lib/karafka/routing/features/active_job/contracts/topic.rb +3 -3
  191. data/lib/karafka/routing/features/base/expander.rb +4 -4
  192. data/lib/karafka/routing/features/base.rb +8 -8
  193. data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +2 -2
  194. data/lib/karafka/routing/features/declaratives/contracts/topic.rb +2 -2
  195. data/lib/karafka/routing/features/deserializers/contracts/topic.rb +2 -2
  196. data/lib/karafka/routing/features/eofed/contracts/topic.rb +3 -3
  197. data/lib/karafka/routing/features/inline_insights/contracts/topic.rb +2 -2
  198. data/lib/karafka/routing/features/inline_insights.rb +7 -7
  199. data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb +2 -2
  200. data/lib/karafka/routing/subscription_group.rb +9 -9
  201. data/lib/karafka/runner.rb +3 -3
  202. data/lib/karafka/server.rb +14 -5
  203. data/lib/karafka/setup/attributes_map.rb +7 -7
  204. data/lib/karafka/setup/config.rb +11 -11
  205. data/lib/karafka/setup/contracts/config.rb +2 -2
  206. data/lib/karafka/setup/defaults_injector.rb +11 -11
  207. data/lib/karafka/swarm/manager.rb +6 -6
  208. data/lib/karafka/swarm/node.rb +8 -37
  209. data/lib/karafka/swarm/producer_replacer.rb +110 -0
  210. data/lib/karafka/swarm/supervisor.rb +9 -6
  211. data/lib/karafka/swarm.rb +1 -1
  212. data/lib/karafka/time_trackers/pause.rb +1 -1
  213. data/lib/karafka/version.rb +1 -1
  214. data/lib/karafka.rb +36 -36
  215. metadata +7 -3
@@ -143,8 +143,8 @@ module Karafka
143
143
  # are always considered immediate as they indicate, that a message with a given key
144
144
  # was already dispatched or that user decided not to dispatch and cancelled the dispatch
145
145
  # via tombstone publishing.
146
- if message.headers['schedule_source_type'] == 'schedule'
147
- time = message.headers['schedule_target_epoch']
146
+ if message.headers["schedule_source_type"] == "schedule"
147
+ time = message.headers["schedule_target_epoch"]
148
148
 
149
149
  # Do not track historical below today as those will be reflected in the daily buffer
150
150
  @tracker.future(message) if time >= @today.starts_at
@@ -162,8 +162,8 @@ module Karafka
162
162
  # that we've got that far in the dispatching time. This allows us (with a certain buffer)
163
163
  # to quickly reject older messages (older in sense of being scheduled for previous times)
164
164
  # instead of loading them into memory until they are expired
165
- if message.headers['schedule_source_type'] == 'tombstone'
166
- @max_epoch.update(message.headers['schedule_target_epoch'])
165
+ if message.headers["schedule_source_type"] == "tombstone"
166
+ @max_epoch.update(message.headers["schedule_target_epoch"])
167
167
  end
168
168
 
169
169
  # Add to buffer all tombstones and messages for the same day
@@ -29,8 +29,8 @@ module Karafka
29
29
  class Config < Karafka::Contracts::Base
30
30
  configure do |config|
31
31
  config.error_messages = YAML.safe_load_file(
32
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
33
- ).fetch('en').fetch('validations').fetch('setup').fetch('config')
32
+ File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
33
+ ).fetch("en").fetch("validations").fetch("setup").fetch("config")
34
34
  end
35
35
 
36
36
  nested(:scheduled_messages) do
@@ -31,8 +31,8 @@ module Karafka
31
31
  class Message < Karafka::Contracts::Base
32
32
  configure do |config|
33
33
  config.error_messages = YAML.safe_load_file(
34
- File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
35
- ).fetch('en').fetch('validations').fetch('scheduled_messages_message')
34
+ File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
35
+ ).fetch("en").fetch("validations").fetch("scheduled_messages_message")
36
36
  end
37
37
 
38
38
  # Headers we expect in each message of type "schedule" that goes to our scheduled
@@ -56,7 +56,7 @@ module Karafka
56
56
  required(:headers) do |val|
57
57
  next false unless val.is_a?(Hash)
58
58
 
59
- if val['schedule_source_type'] == 'message'
59
+ if val["schedule_source_type"] == "message"
60
60
  (val.keys & EXPECTED_SCHEDULE_HEADERS).size >= 4
61
61
  else
62
62
  (val.keys & EXPECTED_CANCEL_HEADERS).size >= 2
@@ -70,9 +70,9 @@ module Karafka
70
70
  next unless errors.empty?
71
71
 
72
72
  # Validate epoch only for schedules
73
- next unless data[:headers]['schedule_source_type'] == 'schedule'
73
+ next unless data[:headers]["schedule_source_type"] == "schedule"
74
74
 
75
- epoch_time = data[:headers].fetch('schedule_target_epoch').to_i
75
+ epoch_time = data[:headers].fetch("schedule_target_epoch").to_i
76
76
 
77
77
  # We allow for small lag as those will be dispatched but we should prevent dispatching
78
78
  # in the past in general as often it is a source of errors
@@ -86,11 +86,11 @@ module Karafka
86
86
  next unless errors.empty?
87
87
 
88
88
  scheduled_topics = Karafka::App
89
- .routes
90
- .flat_map(&:topics)
91
- .flat_map(&:to_a)
92
- .select(&:scheduled_messages?)
93
- .map(&:name)
89
+ .routes
90
+ .flat_map(&:topics)
91
+ .flat_map(&:to_a)
92
+ .select(&:scheduled_messages?)
93
+ .map(&:name)
94
94
 
95
95
  next if scheduled_topics.include?(data[:topic].to_s)
96
96
 
@@ -49,12 +49,12 @@ module Karafka
49
49
  # @note Only messages for a given day should be added here.
50
50
  def <<(message)
51
51
  # Non schedule are only tombstones and cancellations
52
- schedule = message.headers['schedule_source_type'] == 'schedule'
52
+ schedule = message.headers["schedule_source_type"] == "schedule"
53
53
 
54
54
  key = message.key
55
55
 
56
56
  if schedule
57
- epoch = message.headers['schedule_target_epoch']
57
+ epoch = message.headers["schedule_target_epoch"]
58
58
  @accu[key] = [epoch, message]
59
59
  else
60
60
  @accu.delete(key)
@@ -38,18 +38,18 @@ module Karafka
38
38
  def call(metadata)
39
39
  raw_headers = metadata.raw_headers
40
40
 
41
- type = raw_headers.fetch('schedule_source_type')
41
+ type = raw_headers.fetch("schedule_source_type")
42
42
 
43
43
  # tombstone and cancellation events are not operable, thus we do not have to cast any
44
44
  # of the headers pieces
45
45
  return raw_headers unless WORKABLE_TYPES.include?(type)
46
46
 
47
47
  headers = raw_headers.dup
48
- headers['schedule_target_epoch'] = headers['schedule_target_epoch'].to_i
48
+ headers["schedule_target_epoch"] = headers["schedule_target_epoch"].to_i
49
49
 
50
50
  # This attribute is optional, this is why we have to check for its existence
51
- if headers.key?('schedule_target_partition')
52
- headers['schedule_target_partition'] = headers['schedule_target_partition'].to_i
51
+ if headers.key?("schedule_target_partition")
52
+ headers["schedule_target_partition"] = headers["schedule_target_partition"].to_i
53
53
  end
54
54
 
55
55
  headers
@@ -54,10 +54,10 @@ module Karafka
54
54
  # @note It also produces needed tombstone event as well as an audit log message
55
55
  def <<(message)
56
56
  target_headers = message.raw_headers.merge(
57
- 'schedule_source_topic' => @topic,
58
- 'schedule_source_partition' => @partition.to_s,
59
- 'schedule_source_offset' => message.offset.to_s,
60
- 'schedule_source_key' => message.key
57
+ "schedule_source_topic" => @topic,
58
+ "schedule_source_partition" => @partition.to_s,
59
+ "schedule_source_offset" => message.offset.to_s,
60
+ "schedule_source_key" => message.key
61
61
  ).compact
62
62
 
63
63
  target = {
@@ -90,7 +90,7 @@ module Karafka
90
90
  # We use the state as a key, so we always have one state transition data available
91
91
  key: "#{tracker.state}_state",
92
92
  partition: @partition,
93
- headers: { 'zlib' => 'true' }
93
+ headers: { "zlib" => "true" }
94
94
  )
95
95
  end
96
96
 
@@ -84,9 +84,9 @@ module Karafka
84
84
  )
85
85
 
86
86
  headers = (message[:headers] || {}).merge(
87
- 'schedule_schema_version' => ScheduledMessages::SCHEMA_VERSION,
88
- 'schedule_target_epoch' => epoch.to_i.to_s,
89
- 'schedule_source_type' => 'schedule'
87
+ "schedule_schema_version" => ScheduledMessages::SCHEMA_VERSION,
88
+ "schedule_target_epoch" => epoch.to_i.to_s,
89
+ "schedule_source_type" => "schedule"
90
90
  )
91
91
 
92
92
  export(headers, message, :topic)
@@ -118,8 +118,8 @@ module Karafka
118
118
  key: key,
119
119
  payload: nil,
120
120
  headers: {
121
- 'schedule_schema_version' => ScheduledMessages::SCHEMA_VERSION,
122
- 'schedule_source_type' => 'cancel'
121
+ "schedule_schema_version" => ScheduledMessages::SCHEMA_VERSION,
122
+ "schedule_source_type" => "cancel"
123
123
  }
124
124
  }.merge(envelope)
125
125
 
@@ -140,9 +140,9 @@ module Karafka
140
140
  topic: message.topic,
141
141
  partition: message.partition,
142
142
  headers: message.raw_headers.merge(
143
- 'schedule_schema_version' => ScheduledMessages::SCHEMA_VERSION,
144
- 'schedule_source_type' => 'tombstone',
145
- 'schedule_source_offset' => message.offset.to_s
143
+ "schedule_schema_version" => ScheduledMessages::SCHEMA_VERSION,
144
+ "schedule_source_type" => "tombstone",
145
+ "schedule_source_offset" => message.offset.to_s
146
146
  )
147
147
  }
148
148
  end
@@ -33,7 +33,7 @@ module Karafka
33
33
  #
34
34
  # @param message [Karafka::Messages::Message]
35
35
  def call(message)
36
- message_version = message.headers['schedule_schema_version']
36
+ message_version = message.headers["schedule_schema_version"]
37
37
 
38
38
  return if message_version <= ScheduledMessages::SCHEMA_VERSION
39
39
 
@@ -30,7 +30,7 @@ module Karafka
30
30
  extend Karafka::Core::Configurable
31
31
 
32
32
  setting(:consumer_class, default: Consumer)
33
- setting(:group_id, default: 'karafka_scheduled_messages')
33
+ setting(:group_id, default: "karafka_scheduled_messages")
34
34
 
35
35
  # By default we will run the scheduling every 15 seconds since we provide a minute-based
36
36
  # precision. Can be increased when having dedicated processes to run this. Lower values
@@ -52,7 +52,7 @@ module Karafka
52
52
  setting(:dispatcher_class, default: Dispatcher)
53
53
 
54
54
  # Postfix for the states topic to build the states based on the group name + postfix
55
- setting(:states_postfix, default: '_states')
55
+ setting(:states_postfix, default: "_states")
56
56
 
57
57
  setting(:deserializers) do
58
58
  # Deserializer for schedules messages to convert epochs
@@ -46,7 +46,7 @@ module Karafka
46
46
 
47
47
  # Initializes the state as fresh
48
48
  def initialize
49
- @state = 'fresh'
49
+ @state = "fresh"
50
50
  end
51
51
 
52
52
  STATES.each do |state|
@@ -40,7 +40,7 @@ module Karafka
40
40
  @daily = Hash.new { |h, k| h[k] = 0 }
41
41
  @started_at = Time.now.to_i
42
42
  @offsets = { low: -1, high: -1 }
43
- @state = 'fresh'
43
+ @state = "fresh"
44
44
  @reloads = 0
45
45
  end
46
46
 
@@ -71,7 +71,7 @@ module Karafka
71
71
  # drastically increase complexity. For given day we use the accurate counter and for
72
72
  # future days we use estimates.
73
73
  def future(message)
74
- epoch = message.headers['schedule_target_epoch']
74
+ epoch = message.headers["schedule_target_epoch"]
75
75
 
76
76
  @daily[epoch_to_date(epoch)] += 1
77
77
  end
@@ -33,11 +33,11 @@ module Karafka
33
33
  # Version of the schema we use for envelops in scheduled messages.
34
34
  # We use it to detect any potential upgrades similar to other components of Karafka and to
35
35
  # stop processing of incompatible versions
36
- SCHEMA_VERSION = '1.0.0'
36
+ SCHEMA_VERSION = "1.0.0"
37
37
 
38
38
  # Version of the states schema. Used to publish per partition simple aggregated metrics
39
39
  # that can be used for schedules reporting
40
- STATES_SCHEMA_VERSION = '1.0.0'
40
+ STATES_SCHEMA_VERSION = "1.0.0"
41
41
 
42
42
  class << self
43
43
  # Runs the `Proxy.call`
@@ -181,7 +181,7 @@ module Karafka
181
181
 
182
182
  # @return [Integer] RSS in MB for the current process
183
183
  def rss_mb
184
- RUBY_PLATFORM.include?('linux') ? rss_mb_linux : rss_mb_macos
184
+ RUBY_PLATFORM.include?("linux") ? rss_mb_linux : rss_mb_macos
185
185
  end
186
186
 
187
187
  # @return [Integer] RSS in MB for the current process on Linux
@@ -189,7 +189,7 @@ module Karafka
189
189
  kb_rss = 0
190
190
 
191
191
  File.readlines("/proc/#{node.pid}/status").each do |line|
192
- next unless line.start_with?('VmRSS:')
192
+ next unless line.start_with?("VmRSS:")
193
193
 
194
194
  kb_rss = line.split[1].to_i
195
195
  break
@@ -98,7 +98,7 @@ module Karafka
98
98
  # Informs monitoring about trapped signal
99
99
  # @param signal [Symbol] signal type that we received
100
100
  def notice_signal(signal)
101
- Karafka.monitor.instrument('process.notice_signal', caller: self, signal: signal)
101
+ Karafka.monitor.instrument("process.notice_signal", caller: self, signal: signal)
102
102
  end
103
103
  end
104
104
  end
@@ -100,7 +100,7 @@ module Karafka
100
100
 
101
101
  # This should never happen. If it does, something is heavily out of sync. Please reach
102
102
  # out to us if you encounter this
103
- raise Karafka::Errors::InvalidCoordinatorStateError, 'Was zero before decrementation'
103
+ raise Karafka::Errors::InvalidCoordinatorStateError, "Was zero before decrementation"
104
104
  end
105
105
  end
106
106
 
@@ -33,10 +33,10 @@ module Karafka
33
33
  !insights.empty?
34
34
  end
35
35
 
36
- alias statistics insights
37
- alias statistics? insights?
38
- alias inline_insights insights
39
- alias inline_insights? insights?
36
+ alias_method :statistics, :insights
37
+ alias_method :statistics?, :insights?
38
+ alias_method :inline_insights, :insights
39
+ alias_method :inline_insights?, :insights?
40
40
  end
41
41
  end
42
42
  end
@@ -68,8 +68,8 @@ module Karafka
68
68
  # @param statistics [Hash] librdkafka enriched statistics
69
69
  def add(consumer_group_id, statistics)
70
70
  @mutex.synchronize do
71
- statistics.fetch('topics', EMPTY_HASH).each do |topic_name, t_details|
72
- t_details.fetch('partitions', EMPTY_HASH).each do |partition_id, p_details|
71
+ statistics.fetch("topics", EMPTY_HASH).each do |topic_name, t_details|
72
+ t_details.fetch("partitions", EMPTY_HASH).each do |partition_id, p_details|
73
73
  next unless track?(partition_id, p_details)
74
74
 
75
75
  key = "#{consumer_group_id}_#{topic_name}_#{partition_id}"
@@ -114,12 +114,12 @@ module Karafka
114
114
  # @param p_details [Hash] partition statistics details
115
115
  # @return [Boolean] true if we should track given partition
116
116
  def track?(partition_id, p_details)
117
- return false if partition_id == '-1'
117
+ return false if partition_id == "-1"
118
118
 
119
- fetch_state = p_details.fetch('fetch_state')
119
+ fetch_state = p_details.fetch("fetch_state")
120
120
 
121
- return false if fetch_state == 'stopped'
122
- return false if fetch_state == 'none'
121
+ return false if fetch_state == "stopped"
122
+ return false if fetch_state == "none"
123
123
 
124
124
  true
125
125
  end
@@ -38,19 +38,21 @@ module Karafka
38
38
  # When redefined can run any code prior to the job being scheduled
39
39
  # @note This will run in the listener thread and not in the worker
40
40
  def before_schedule
41
- raise NotImplementedError, 'Please implement in a subclass'
41
+ raise NotImplementedError, "Please implement in a subclass"
42
42
  end
43
43
 
44
44
  # When redefined can run any code that should run before executing the proper code
45
- def before_call; end
45
+ def before_call
46
+ end
46
47
 
47
48
  # The main entry-point of a job
48
49
  def call
49
- raise NotImplementedError, 'Please implement in a subclass'
50
+ raise NotImplementedError, "Please implement in a subclass"
50
51
  end
51
52
 
52
53
  # When redefined can run any code that should run after executing the proper code
53
- def after_call; end
54
+ def after_call
55
+ end
54
56
 
55
57
  # @return [Boolean] is this a non-blocking job
56
58
  #
@@ -180,6 +180,16 @@ module Karafka
180
180
  end
181
181
  end
182
182
 
183
+ # Returns a snapshot of all jobs currently in processing per group.
184
+ # Useful for diagnostics during forceful shutdown to understand what is blocking.
185
+ #
186
+ # @return [Hash{String => Array<Jobs::Base>}] hash mapping group ids to arrays of jobs
187
+ def in_processing
188
+ @mutex.synchronize do
189
+ @in_processing.transform_values(&:dup).freeze
190
+ end
191
+ end
192
+
183
193
  private
184
194
 
185
195
  # @param group_id [String] id of the group in which jobs we're interested.
@@ -21,10 +21,10 @@ module Karafka
21
21
  end
22
22
 
23
23
  # Revocation, shutdown and idle jobs can also run in fifo by default
24
- alias on_schedule_revocation on_schedule_consumption
25
- alias on_schedule_shutdown on_schedule_consumption
26
- alias on_schedule_idle on_schedule_consumption
27
- alias on_schedule_eofed on_schedule_consumption
24
+ alias_method :on_schedule_revocation, :on_schedule_consumption
25
+ alias_method :on_schedule_shutdown, :on_schedule_consumption
26
+ alias_method :on_schedule_idle, :on_schedule_consumption
27
+ alias_method :on_schedule_eofed, :on_schedule_consumption
28
28
 
29
29
  # This scheduler does not have anything to manage as it is a pass through and has no state
30
30
  def on_manage
@@ -28,32 +28,32 @@ module Karafka
28
28
 
29
29
  # What should happen before we kick in the processing
30
30
  def handle_before_consume
31
- raise NotImplementedError, 'Implement in a subclass'
31
+ raise NotImplementedError, "Implement in a subclass"
32
32
  end
33
33
 
34
34
  # What should happen in the processing
35
35
  def handle_consume
36
- raise NotImplementedError, 'Implement in a subclass'
36
+ raise NotImplementedError, "Implement in a subclass"
37
37
  end
38
38
 
39
39
  # Post-consumption handling
40
40
  def handle_after_consume
41
- raise NotImplementedError, 'Implement in a subclass'
41
+ raise NotImplementedError, "Implement in a subclass"
42
42
  end
43
43
 
44
44
  # Idle run handling
45
45
  def handle_idle
46
- raise NotImplementedError, 'Implement in a subclass'
46
+ raise NotImplementedError, "Implement in a subclass"
47
47
  end
48
48
 
49
49
  # Revocation handling
50
50
  def handle_revoked
51
- raise NotImplementedError, 'Implement in a subclass'
51
+ raise NotImplementedError, "Implement in a subclass"
52
52
  end
53
53
 
54
54
  # Shutdown handling
55
55
  def handle_shutdown
56
- raise NotImplementedError, 'Implement in a subclass'
56
+ raise NotImplementedError, "Implement in a subclass"
57
57
  end
58
58
  end
59
59
  end
@@ -35,8 +35,8 @@ module Karafka
35
35
  # @note It runs in the listener loop. Should **not** be used for anything heavy or
36
36
  # with any potential errors. Mostly for initialization of states, etc.
37
37
  def handle_initialized
38
- monitor.instrument('consumer.initialize', caller: self)
39
- monitor.instrument('consumer.initialized', caller: self) do
38
+ monitor.instrument("consumer.initialize", caller: self)
39
+ monitor.instrument("consumer.initialized", caller: self) do
40
40
  initialized
41
41
  end
42
42
  end
@@ -120,22 +120,22 @@ module Karafka
120
120
  # @param action [Symbol]
121
121
  # @param block [Proc]
122
122
  def handle_wrap(action, &block)
123
- monitor.instrument('consumer.wrap', caller: self)
124
- monitor.instrument('consumer.wrapped', caller: self) do
123
+ monitor.instrument("consumer.wrap", caller: self)
124
+ monitor.instrument("consumer.wrapped", caller: self) do
125
125
  wrap(action, &block)
126
126
  end
127
127
  end
128
128
 
129
129
  # Run the user consumption code
130
130
  def handle_consume
131
- monitor.instrument('consumer.consume', caller: self)
132
- monitor.instrument('consumer.consumed', caller: self) do
131
+ monitor.instrument("consumer.consume", caller: self)
132
+ monitor.instrument("consumer.consumed", caller: self) do
133
133
  consume
134
134
  end
135
135
 
136
136
  # Mark job as successful
137
137
  coordinator.success!(self)
138
- rescue StandardError => e
138
+ rescue => e
139
139
  coordinator.failure!(self, e)
140
140
 
141
141
  # Re-raise so reported in the consumer
@@ -175,8 +175,8 @@ module Karafka
175
175
 
176
176
  # Runs the consumer `#eofed` method with reporting
177
177
  def handle_eofed
178
- monitor.instrument('consumer.eof', caller: self)
179
- monitor.instrument('consumer.eofed', caller: self) do
178
+ monitor.instrument("consumer.eof", caller: self)
179
+ monitor.instrument("consumer.eofed", caller: self) do
180
180
  eofed
181
181
  end
182
182
  ensure
@@ -191,8 +191,8 @@ module Karafka
191
191
 
192
192
  coordinator.revoke
193
193
 
194
- monitor.instrument('consumer.revoke', caller: self)
195
- monitor.instrument('consumer.revoked', caller: self) do
194
+ monitor.instrument("consumer.revoke", caller: self)
195
+ monitor.instrument("consumer.revoked", caller: self) do
196
196
  revoked
197
197
  end
198
198
  ensure
@@ -201,8 +201,8 @@ module Karafka
201
201
 
202
202
  # Runs the shutdown code
203
203
  def handle_shutdown
204
- monitor.instrument('consumer.shutting_down', caller: self)
205
- monitor.instrument('consumer.shutdown', caller: self) do
204
+ monitor.instrument("consumer.shutting_down", caller: self)
205
+ monitor.instrument("consumer.shutdown", caller: self) do
206
206
  shutdown
207
207
  end
208
208
  ensure
@@ -118,7 +118,7 @@ module Karafka
118
118
 
119
119
  # Notify about dispatch on the events bus
120
120
  monitor.instrument(
121
- 'dead_letter_queue.dispatched',
121
+ "dead_letter_queue.dispatched",
122
122
  caller: self,
123
123
  message: skippable_message
124
124
  )
@@ -55,9 +55,9 @@ module Karafka
55
55
 
56
56
  if job
57
57
  job.wrap do
58
- monitor.instrument('worker.process', instrument_details)
58
+ monitor.instrument("worker.process", instrument_details)
59
59
 
60
- monitor.instrument('worker.processed', instrument_details) do
60
+ monitor.instrument("worker.processed", instrument_details) do
61
61
  job.before_call
62
62
 
63
63
  # If a job is marked as non blocking, we can run a tick in the job queue and if there
@@ -87,12 +87,12 @@ module Karafka
87
87
  rescue Exception => e
88
88
  # rubocop:enable Lint/RescueException
89
89
  monitor.instrument(
90
- 'error.occurred',
90
+ "error.occurred",
91
91
  caller: self,
92
92
  job: job,
93
93
  jobs_queue: @jobs_queue,
94
94
  error: e,
95
- type: 'worker.process.error'
95
+ type: "worker.process.error"
96
96
  )
97
97
  ensure
98
98
  # job can be nil when the queue is being closed
@@ -102,7 +102,7 @@ module Karafka
102
102
  end
103
103
 
104
104
  # Always publish info, that we completed all the work despite its result
105
- monitor.instrument('worker.completed', instrument_details)
105
+ monitor.instrument("worker.completed", instrument_details)
106
106
  end
107
107
  end
108
108
  end
@@ -12,17 +12,17 @@ end
12
12
 
13
13
  if Karafka.rails?
14
14
  # Load ActiveJob adapter
15
- require 'active_job/karafka'
15
+ require "active_job/karafka"
16
16
 
17
17
  # Setup env if configured (may be configured later by .net, etc)
18
- ENV['KARAFKA_ENV'] ||= ENV['RAILS_ENV'] if ENV.key?('RAILS_ENV')
18
+ ENV["KARAFKA_ENV"] ||= ENV["RAILS_ENV"] if ENV.key?("RAILS_ENV")
19
19
 
20
20
  module Karafka
21
21
  # Railtie for setting up Rails integration
22
22
  class Railtie < Rails::Railtie
23
23
  railtie_name :karafka
24
24
 
25
- initializer 'karafka.active_job_integration' do
25
+ initializer "karafka.active_job_integration" do
26
26
  ActiveSupport.on_load(:active_job) do
27
27
  # Extend ActiveJob with some Karafka specific ActiveJob magic
28
28
  extend ::Karafka::ActiveJob::JobExtensions
@@ -32,12 +32,12 @@ if Karafka.rails?
32
32
  # This lines will make Karafka print to stdout like puma or unicorn when we run karafka
33
33
  # server + will support code reloading with each fetched loop. We do it only for karafka
34
34
  # based commands as Rails processes and console will have it enabled already
35
- initializer 'karafka.configure_rails_logger' do
35
+ initializer "karafka.configure_rails_logger" do
36
36
  # Make Karafka uses Rails logger
37
37
  ::Karafka::App.config.logger = Rails.logger
38
38
 
39
39
  next unless Rails.env.development?
40
- next unless ENV.key?('KARAFKA_CLI')
40
+ next unless ENV.key?("KARAFKA_CLI")
41
41
  # If we are already publishing to STDOUT, no need to add it again.
42
42
  # If added again, would print stuff twice
43
43
  next if ActiveSupport::Logger.logger_outputs_to?(Rails.logger, $stdout)
@@ -46,7 +46,7 @@ if Karafka.rails?
46
46
  # Inherit the logger level from Rails, otherwise would always run with the debug level
47
47
  stdout_logger.level = Rails.logger.level
48
48
 
49
- rails71plus = Rails.gem_version >= Gem::Version.new('7.1.0')
49
+ rails71plus = Rails.gem_version >= Gem::Version.new("7.1.0")
50
50
 
51
51
  # Rails 7.1 replaced the broadcast module with a broadcast logger
52
52
  # While 7.1 is EOL, we keep this for users who may still use it without official support
@@ -61,20 +61,20 @@ if Karafka.rails?
61
61
  end
62
62
  end
63
63
 
64
- initializer 'karafka.configure_rails_auto_load_paths' do |app|
64
+ initializer "karafka.configure_rails_auto_load_paths" do |app|
65
65
  # Consumers should autoload by default in the Rails app so they are visible
66
66
  app.config.autoload_paths += %w[app/consumers]
67
67
  end
68
68
 
69
- initializer 'karafka.require_karafka_boot_file' do |app|
70
- rails6plus = Rails.gem_version >= Gem::Version.new('6.0.0')
69
+ initializer "karafka.require_karafka_boot_file" do |app|
70
+ rails6plus = Rails.gem_version >= Gem::Version.new("6.0.0")
71
71
 
72
72
  # If the boot file location is set to "false", we should not raise an exception and we
73
73
  # should just not load karafka stuff. Setting this explicitly to false indicates, that
74
74
  # karafka is part of the supply chain but it is not a first class citizen of a given
75
75
  # system (may be just a dependency of a dependency), thus railtie should not kick in to
76
76
  # load the non-existing boot file
77
- next if Karafka.boot_file.to_s == 'false'
77
+ next if Karafka.boot_file.to_s == "false"
78
78
 
79
79
  karafka_boot_file = Rails.root.join(Karafka.boot_file.to_s).to_s
80
80
 
@@ -96,7 +96,7 @@ if Karafka.rails?
96
96
  end
97
97
  end
98
98
 
99
- initializer 'karafka.configure_worker_external_executor' do |app|
99
+ initializer "karafka.configure_worker_external_executor" do |app|
100
100
  app.config.after_initialize do
101
101
  app_config = Karafka::App.config
102
102