karafka 2.5.1 → 2.5.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (238) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ci_linux_ubuntu_x86_64_gnu.yml +21 -29
  3. data/.github/workflows/ci_macos_arm64.yml +1 -1
  4. data/.github/workflows/push.yml +2 -2
  5. data/.github/workflows/trigger-wiki-refresh.yml +1 -1
  6. data/.ruby-version +1 -1
  7. data/.yard-lint.yml +174 -0
  8. data/CHANGELOG.md +20 -4
  9. data/Gemfile +1 -2
  10. data/Gemfile.lock +45 -41
  11. data/bin/integrations +2 -1
  12. data/bin/rspecs +4 -0
  13. data/config/locales/errors.yml +6 -4
  14. data/config/locales/pro_errors.yml +5 -4
  15. data/docker-compose.yml +1 -1
  16. data/examples/payloads/json/sample_set_02/download.json +191 -0
  17. data/examples/payloads/json/sample_set_03/event_type_1.json +18 -0
  18. data/examples/payloads/json/sample_set_03/event_type_2.json +263 -0
  19. data/examples/payloads/json/sample_set_03/event_type_3.json +41 -0
  20. data/karafka.gemspec +3 -3
  21. data/lib/active_job/queue_adapters/karafka_adapter.rb +3 -3
  22. data/lib/karafka/active_job/consumer.rb +7 -3
  23. data/lib/karafka/active_job/current_attributes/job_wrapper.rb +45 -0
  24. data/lib/karafka/active_job/current_attributes/loading.rb +1 -1
  25. data/lib/karafka/active_job/current_attributes/persistence.rb +19 -7
  26. data/lib/karafka/active_job/current_attributes.rb +3 -2
  27. data/lib/karafka/active_job/deserializer.rb +61 -0
  28. data/lib/karafka/active_job/dispatcher.rb +34 -14
  29. data/lib/karafka/active_job/job_options_contract.rb +2 -4
  30. data/lib/karafka/admin/acl.rb +8 -4
  31. data/lib/karafka/admin/configs/config.rb +6 -4
  32. data/lib/karafka/admin/configs/resource.rb +7 -1
  33. data/lib/karafka/admin/consumer_groups.rb +80 -12
  34. data/lib/karafka/admin/topics.rb +43 -9
  35. data/lib/karafka/admin.rb +23 -14
  36. data/lib/karafka/app.rb +3 -3
  37. data/lib/karafka/base_consumer.rb +6 -6
  38. data/lib/karafka/cli/base.rb +2 -2
  39. data/lib/karafka/cli/console.rb +1 -1
  40. data/lib/karafka/cli/contracts/server.rb +3 -5
  41. data/lib/karafka/cli/help.rb +1 -1
  42. data/lib/karafka/cli/install.rb +3 -2
  43. data/lib/karafka/cli/server.rb +1 -1
  44. data/lib/karafka/cli/swarm.rb +1 -1
  45. data/lib/karafka/cli/topics/align.rb +1 -1
  46. data/lib/karafka/cli/topics/repartition.rb +2 -2
  47. data/lib/karafka/connection/client.rb +30 -19
  48. data/lib/karafka/connection/listeners_batch.rb +2 -3
  49. data/lib/karafka/connection/manager.rb +1 -0
  50. data/lib/karafka/connection/proxy.rb +12 -8
  51. data/lib/karafka/connection/rebalance_manager.rb +1 -1
  52. data/lib/karafka/connection/status.rb +1 -0
  53. data/lib/karafka/constraints.rb +1 -1
  54. data/lib/karafka/contracts/base.rb +1 -1
  55. data/lib/karafka/deserializers/payload.rb +1 -1
  56. data/lib/karafka/env.rb +1 -2
  57. data/lib/karafka/helpers/async.rb +1 -1
  58. data/lib/karafka/helpers/config_importer.rb +3 -3
  59. data/lib/karafka/helpers/interval_runner.rb +4 -1
  60. data/lib/karafka/helpers/multi_delegator.rb +3 -0
  61. data/lib/karafka/instrumentation/assignments_tracker.rb +19 -1
  62. data/lib/karafka/instrumentation/callbacks/error.rb +2 -2
  63. data/lib/karafka/instrumentation/callbacks/statistics.rb +3 -3
  64. data/lib/karafka/instrumentation/logger.rb +6 -6
  65. data/lib/karafka/instrumentation/monitor.rb +3 -3
  66. data/lib/karafka/instrumentation/notifications.rb +1 -0
  67. data/lib/karafka/instrumentation/vendors/appsignal/base.rb +3 -4
  68. data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +3 -4
  69. data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +10 -11
  70. data/lib/karafka/instrumentation/vendors/kubernetes/base_listener.rb +1 -1
  71. data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +5 -18
  72. data/lib/karafka/messages/builders/batch_metadata.rb +2 -2
  73. data/lib/karafka/messages/builders/message.rb +1 -1
  74. data/lib/karafka/messages/messages.rb +2 -3
  75. data/lib/karafka/patches/rdkafka/bindings.rb +6 -6
  76. data/lib/karafka/patches/rdkafka/opaque.rb +1 -1
  77. data/lib/karafka/pro/active_job/consumer.rb +2 -2
  78. data/lib/karafka/pro/active_job/dispatcher.rb +10 -6
  79. data/lib/karafka/pro/active_job/job_options_contract.rb +2 -4
  80. data/lib/karafka/pro/cleaner/messages/messages.rb +2 -3
  81. data/lib/karafka/pro/cleaner.rb +3 -3
  82. data/lib/karafka/pro/cli/contracts/server.rb +3 -5
  83. data/lib/karafka/pro/cli/parallel_segments/base.rb +5 -5
  84. data/lib/karafka/pro/cli/parallel_segments/collapse.rb +3 -3
  85. data/lib/karafka/pro/cli/parallel_segments/distribute.rb +3 -3
  86. data/lib/karafka/pro/cli/parallel_segments.rb +1 -1
  87. data/lib/karafka/pro/connection/manager.rb +3 -4
  88. data/lib/karafka/pro/connection/multiplexing/listener.rb +1 -0
  89. data/lib/karafka/pro/contracts/base.rb +1 -1
  90. data/lib/karafka/pro/encryption/cipher.rb +3 -2
  91. data/lib/karafka/pro/encryption/contracts/config.rb +5 -7
  92. data/lib/karafka/pro/encryption/messages/parser.rb +4 -4
  93. data/lib/karafka/pro/encryption/setup/config.rb +1 -1
  94. data/lib/karafka/pro/instrumentation/performance_tracker.rb +3 -3
  95. data/lib/karafka/pro/iterator/expander.rb +1 -1
  96. data/lib/karafka/pro/iterator/tpl_builder.rb +2 -2
  97. data/lib/karafka/pro/iterator.rb +3 -3
  98. data/lib/karafka/pro/loader.rb +1 -1
  99. data/lib/karafka/pro/processing/coordinator.rb +1 -1
  100. data/lib/karafka/pro/processing/coordinators/errors_tracker.rb +2 -3
  101. data/lib/karafka/pro/processing/coordinators/filters_applier.rb +3 -3
  102. data/lib/karafka/pro/processing/filters/base.rb +1 -0
  103. data/lib/karafka/pro/processing/filters/delayer.rb +1 -1
  104. data/lib/karafka/pro/processing/filters/expirer.rb +1 -1
  105. data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb +1 -1
  106. data/lib/karafka/pro/processing/filters/throttler.rb +1 -1
  107. data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +1 -1
  108. data/lib/karafka/pro/processing/jobs/eofed_non_blocking.rb +1 -1
  109. data/lib/karafka/pro/processing/jobs/periodic.rb +1 -1
  110. data/lib/karafka/pro/processing/jobs/revoked_non_blocking.rb +1 -1
  111. data/lib/karafka/pro/processing/jobs_builder.rb +1 -1
  112. data/lib/karafka/pro/processing/offset_metadata/fetcher.rb +1 -0
  113. data/lib/karafka/pro/processing/partitioner.rb +1 -1
  114. data/lib/karafka/pro/processing/schedulers/default.rb +2 -4
  115. data/lib/karafka/pro/processing/strategies/base.rb +1 -1
  116. data/lib/karafka/pro/processing/strategies/default.rb +2 -2
  117. data/lib/karafka/pro/processing/strategies/lrj/default.rb +2 -4
  118. data/lib/karafka/pro/processing/strategies/vp/default.rb +2 -4
  119. data/lib/karafka/pro/processing/strategy_selector.rb +1 -0
  120. data/lib/karafka/pro/processing/subscription_groups_coordinator.rb +2 -3
  121. data/lib/karafka/pro/processing/virtual_partitions/distributors/balanced.rb +4 -2
  122. data/lib/karafka/pro/processing/virtual_partitions/distributors/consistent.rb +4 -2
  123. data/lib/karafka/pro/recurring_tasks/consumer.rb +3 -2
  124. data/lib/karafka/pro/recurring_tasks/contracts/config.rb +4 -6
  125. data/lib/karafka/pro/recurring_tasks/contracts/task.rb +3 -5
  126. data/lib/karafka/pro/recurring_tasks/deserializer.rb +1 -1
  127. data/lib/karafka/pro/recurring_tasks/dispatcher.rb +7 -6
  128. data/lib/karafka/pro/recurring_tasks/executor.rb +2 -1
  129. data/lib/karafka/pro/recurring_tasks/schedule.rb +9 -8
  130. data/lib/karafka/pro/recurring_tasks/serializer.rb +6 -5
  131. data/lib/karafka/pro/recurring_tasks/setup/config.rb +2 -2
  132. data/lib/karafka/pro/recurring_tasks/task.rb +1 -1
  133. data/lib/karafka/pro/recurring_tasks.rb +8 -5
  134. data/lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb +2 -4
  135. data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +2 -4
  136. data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb +3 -0
  137. data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb +2 -4
  138. data/lib/karafka/pro/routing/features/delaying/topic.rb +2 -4
  139. data/lib/karafka/pro/routing/features/direct_assignments/contracts/consumer_group.rb +4 -8
  140. data/lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb +5 -7
  141. data/lib/karafka/pro/routing/features/direct_assignments/subscription_group.rb +7 -6
  142. data/lib/karafka/pro/routing/features/direct_assignments/topic.rb +2 -2
  143. data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb +2 -4
  144. data/lib/karafka/pro/routing/features/expiring/topic.rb +2 -4
  145. data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb +2 -4
  146. data/lib/karafka/pro/routing/features/filtering/topic.rb +2 -3
  147. data/lib/karafka/pro/routing/features/inline_insights/contracts/topic.rb +2 -4
  148. data/lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb +2 -4
  149. data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb +3 -5
  150. data/lib/karafka/pro/routing/features/multiplexing/subscription_groups_builder.rb +1 -1
  151. data/lib/karafka/pro/routing/features/multiplexing.rb +5 -5
  152. data/lib/karafka/pro/routing/features/non_blocking_job/topic.rb +3 -3
  153. data/lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb +2 -4
  154. data/lib/karafka/pro/routing/features/offset_metadata.rb +4 -4
  155. data/lib/karafka/pro/routing/features/parallel_segments/builder.rb +1 -1
  156. data/lib/karafka/pro/routing/features/parallel_segments/contracts/consumer_group.rb +2 -4
  157. data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +3 -5
  158. data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb +2 -4
  159. data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +2 -4
  160. data/lib/karafka/pro/routing/features/patterns/patterns.rb +1 -1
  161. data/lib/karafka/pro/routing/features/pausing/config.rb +26 -0
  162. data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb +17 -11
  163. data/lib/karafka/pro/routing/features/pausing/topic.rb +69 -8
  164. data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb +2 -4
  165. data/lib/karafka/pro/routing/features/periodic_job/topic.rb +1 -1
  166. data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb +1 -1
  167. data/lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb +2 -4
  168. data/lib/karafka/pro/routing/features/scheduled_messages/contracts/topic.rb +2 -4
  169. data/lib/karafka/pro/routing/features/swarm/contracts/routing.rb +2 -4
  170. data/lib/karafka/pro/routing/features/swarm/contracts/topic.rb +6 -8
  171. data/lib/karafka/pro/routing/features/swarm.rb +1 -1
  172. data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb +2 -4
  173. data/lib/karafka/pro/routing/features/throttling/topic.rb +3 -1
  174. data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +2 -4
  175. data/lib/karafka/pro/scheduled_messages/consumer.rb +1 -1
  176. data/lib/karafka/pro/scheduled_messages/contracts/config.rb +4 -6
  177. data/lib/karafka/pro/scheduled_messages/contracts/message.rb +3 -5
  178. data/lib/karafka/pro/scheduled_messages/daily_buffer.rb +3 -2
  179. data/lib/karafka/pro/scheduled_messages/day.rb +1 -0
  180. data/lib/karafka/pro/scheduled_messages/deserializers/headers.rb +1 -1
  181. data/lib/karafka/pro/scheduled_messages/deserializers/payload.rb +1 -1
  182. data/lib/karafka/pro/scheduled_messages/max_epoch.rb +1 -0
  183. data/lib/karafka/pro/scheduled_messages/proxy.rb +1 -1
  184. data/lib/karafka/pro/scheduled_messages/serializer.rb +3 -3
  185. data/lib/karafka/pro/scheduled_messages/setup/config.rb +2 -2
  186. data/lib/karafka/pro/scheduled_messages/state.rb +1 -0
  187. data/lib/karafka/pro/scheduled_messages/tracker.rb +1 -0
  188. data/lib/karafka/pro/scheduled_messages.rb +4 -6
  189. data/lib/karafka/pro/swarm/liveness_listener.rb +2 -2
  190. data/lib/karafka/process.rb +4 -4
  191. data/lib/karafka/processing/coordinator.rb +2 -4
  192. data/lib/karafka/processing/coordinators_buffer.rb +2 -3
  193. data/lib/karafka/processing/executor.rb +3 -4
  194. data/lib/karafka/processing/inline_insights/tracker.rb +1 -0
  195. data/lib/karafka/processing/jobs/base.rb +2 -3
  196. data/lib/karafka/processing/jobs_queue.rb +1 -1
  197. data/lib/karafka/processing/result.rb +1 -0
  198. data/lib/karafka/processing/strategy_selector.rb +1 -0
  199. data/lib/karafka/processing/workers_batch.rb +2 -3
  200. data/lib/karafka/railtie.rb +1 -0
  201. data/lib/karafka/routing/activity_manager.rb +3 -2
  202. data/lib/karafka/routing/builder.rb +8 -8
  203. data/lib/karafka/routing/consumer_group.rb +4 -6
  204. data/lib/karafka/routing/contracts/consumer_group.rb +6 -7
  205. data/lib/karafka/routing/contracts/routing.rb +2 -4
  206. data/lib/karafka/routing/contracts/topic.rb +7 -6
  207. data/lib/karafka/routing/features/active_job/contracts/topic.rb +2 -4
  208. data/lib/karafka/routing/features/active_job/topic.rb +6 -0
  209. data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +3 -5
  210. data/lib/karafka/routing/features/declaratives/contracts/topic.rb +3 -5
  211. data/lib/karafka/routing/features/declaratives/topic.rb +5 -2
  212. data/lib/karafka/routing/features/deserializers/contracts/topic.rb +2 -4
  213. data/lib/karafka/routing/features/deserializers/topic.rb +3 -3
  214. data/lib/karafka/routing/features/eofed/contracts/topic.rb +2 -4
  215. data/lib/karafka/routing/features/inline_insights/contracts/topic.rb +2 -4
  216. data/lib/karafka/routing/features/inline_insights.rb +5 -5
  217. data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb +2 -4
  218. data/lib/karafka/routing/router.rb +1 -1
  219. data/lib/karafka/routing/subscription_group.rb +1 -1
  220. data/lib/karafka/routing/subscription_groups_builder.rb +1 -0
  221. data/lib/karafka/routing/topic.rb +3 -3
  222. data/lib/karafka/routing/topics.rb +4 -9
  223. data/lib/karafka/server.rb +2 -2
  224. data/lib/karafka/setup/attributes_map.rb +4 -2
  225. data/lib/karafka/setup/config.rb +85 -17
  226. data/lib/karafka/setup/config_proxy.rb +209 -0
  227. data/lib/karafka/setup/contracts/config.rb +13 -11
  228. data/lib/karafka/setup/defaults_injector.rb +3 -2
  229. data/lib/karafka/setup/dsl.rb +2 -3
  230. data/lib/karafka/swarm/liveness_listener.rb +3 -3
  231. data/lib/karafka/swarm/manager.rb +7 -6
  232. data/lib/karafka/swarm/node.rb +1 -1
  233. data/lib/karafka/swarm/supervisor.rb +2 -1
  234. data/lib/karafka/time_trackers/base.rb +1 -1
  235. data/lib/karafka/version.rb +1 -1
  236. data/lib/karafka.rb +4 -4
  237. metadata +14 -6
  238. data/.diffend.yml +0 -3
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'active_support/current_attributes'
4
+ require_relative 'current_attributes/job_wrapper'
4
5
  require_relative 'current_attributes/loading'
5
6
  require_relative 'current_attributes/persistence'
6
7
 
@@ -22,8 +23,8 @@ module Karafka
22
23
  .each { |expandable| expandable.class_attribute :_cattr_klasses, default: {} }
23
24
 
24
25
  # Do not double inject in case of running persist multiple times
25
- Dispatcher.prepend(Persistence) unless Dispatcher.ancestors.include?(Persistence)
26
- Consumer.prepend(Loading) unless Consumer.ancestors.include?(Loading)
26
+ Dispatcher.prepend(Persistence) unless Dispatcher <= Persistence
27
+ Consumer.prepend(Loading) unless Consumer <= Loading
27
28
 
28
29
  klasses.map(&:to_s).each do |stringified_klass|
29
30
  # Prevent registering same klass multiple times
@@ -0,0 +1,61 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module ActiveJob
5
+ # Default deserializer for ActiveJob jobs
6
+ #
7
+ # @note Despite the name, this class handles both serialization (job to Kafka payload) and
8
+ # deserialization (Kafka message to job). It's called "Deserializer" to align with Karafka's
9
+ # naming conventions where message consumption is the primary concern.
10
+ #
11
+ # This class can be inherited and its methods can be overridden to support
12
+ # custom payload formats (e.g., Avro, Protobuf, MessagePack)
13
+ #
14
+ # @example Wrapping jobs in a custom envelope with metadata
15
+ # class EnvelopedJobDeserializer < Karafka::ActiveJob::Deserializer
16
+ # def serialize(job)
17
+ # # Wrap the job in an envelope with additional metadata
18
+ # envelope = {
19
+ # version: 1,
20
+ # produced_at: Time.now.iso8601,
21
+ # producer: 'my-app',
22
+ # payload: job.serialize
23
+ # }
24
+ # ::ActiveSupport::JSON.encode(envelope)
25
+ # end
26
+ #
27
+ # def deserialize(message)
28
+ # # Extract the job from the envelope
29
+ # envelope = ::ActiveSupport::JSON.decode(message.raw_payload)
30
+ #
31
+ # # Could validate envelope version, log metadata, etc.
32
+ # raise 'Unsupported version' if envelope['version'] != 1
33
+ #
34
+ # # Return the actual job data
35
+ # envelope['payload']
36
+ # end
37
+ # end
38
+ #
39
+ # # Configure in Karafka
40
+ # Karafka::App.config.internal.active_job.deserializer = EnvelopedJobDeserializer.new
41
+ class Deserializer
42
+ # Serializes an ActiveJob job into a string payload for Kafka
43
+ #
44
+ # @param job [ActiveJob::Base, #serialize] job to serialize. The job must respond to
45
+ # #serialize which returns a Hash of job attributes. When CurrentAttributes are used,
46
+ # this may be a JobWrapper instance instead of the original ::ActiveJob::Base.
47
+ # @return [String] serialized job payload
48
+ def serialize(job)
49
+ ::ActiveSupport::JSON.encode(job.serialize)
50
+ end
51
+
52
+ # Deserializes a Kafka message payload into an ActiveJob job hash
53
+ #
54
+ # @param message [Karafka::Messages::Message] message containing the job
55
+ # @return [Hash] deserialized job hash
56
+ def deserialize(message)
57
+ ::ActiveSupport::JSON.decode(message.raw_payload)
58
+ end
59
+ end
60
+ end
61
+ end
@@ -4,6 +4,10 @@ module Karafka
4
4
  module ActiveJob
5
5
  # Dispatcher that sends the ActiveJob job to a proper topic based on the queue name
6
6
  class Dispatcher
7
+ include Helpers::ConfigImporter.new(
8
+ deserializer: %i[internal active_job deserializer]
9
+ )
10
+
7
11
  # Defaults for dispatching
8
12
  # The can be updated by using `#karafka_options` on the job
9
13
  DEFAULTS = {
@@ -15,10 +19,10 @@ module Karafka
15
19
 
16
20
  # @param job [ActiveJob::Base] job
17
21
  def dispatch(job)
18
- ::Karafka.producer.public_send(
22
+ Karafka.producer.public_send(
19
23
  fetch_option(job, :dispatch_method, DEFAULTS),
20
24
  topic: job.queue_name,
21
- payload: ::ActiveSupport::JSON.encode(serialize_job(job))
25
+ payload: serialize_job(job)
22
26
  )
23
27
  end
24
28
 
@@ -34,12 +38,12 @@ module Karafka
34
38
 
35
39
  dispatches[d_method] << {
36
40
  topic: job.queue_name,
37
- payload: ::ActiveSupport::JSON.encode(serialize_job(job))
41
+ payload: serialize_job(job)
38
42
  }
39
43
  end
40
44
 
41
45
  dispatches.each do |type, messages|
42
- ::Karafka.producer.public_send(
46
+ Karafka.producer.public_send(
43
47
  type,
44
48
  messages
45
49
  )
@@ -48,14 +52,18 @@ module Karafka
48
52
 
49
53
  # Raises info, that Karafka backend does not support scheduling jobs if someone wants to
50
54
  # schedule jobs in the future. It works for past and present because we want to support
51
- # things like continuation and `#retry_on` API with no wait and no jitter
55
+ # things like continuation and `#retry_on` API with no wait and no jitter.
52
56
  #
53
57
  # @param job [Object] job we cannot enqueue
54
58
  # @param timestamp [Time] time when job should run
55
59
  #
56
- # @note Karafka Pro supports future jobs
60
+ # @note Karafka Pro supports future jobs via the Scheduled Messages feature
61
+ #
62
+ # @note For ActiveJob Continuation to work without Pro, configure your continuable jobs:
63
+ # self.resume_options = { wait: 0 }
57
64
  #
58
- # @note In order for jobs to work with this you need to set jitter to false and no wait
65
+ # @note For `#retry_on` to work without Pro, configure with:
66
+ # retry_on SomeError, wait: 0, jitter: 0
59
67
  def dispatch_at(job, timestamp)
60
68
  # Dispatch at is used by some of the ActiveJob features that actually do not back-off
61
69
  # but things go via this API nonetheless.
@@ -64,13 +72,31 @@ module Karafka
64
72
  else
65
73
  raise NotImplementedError, <<~ERROR_MESSAGE
66
74
  This queueing backend does not support scheduling future jobs.
67
- Consider using Karafka Pro, which supports this via the Scheduled Messages feature.
75
+
76
+ If you're using ActiveJob Continuation, configure your jobs with:
77
+ self.resume_options = { wait: 0 }
78
+
79
+ If you're using retry_on, configure with:
80
+ retry_on SomeError, wait: 0, jitter: 0
81
+
82
+ For full support of delayed job execution, consider using Karafka Pro with Scheduled Messages.
68
83
  ERROR_MESSAGE
69
84
  end
70
85
  end
71
86
 
72
87
  private
73
88
 
89
+ # Serializes a job using the configured deserializer
90
+ # This method serves as an extension point and can be wrapped by modules like
91
+ # CurrentAttributes::Persistence
92
+ #
93
+ # @param job [ActiveJob::Base, CurrentAttributes::Persistence::JobWrapper] job to serialize.
94
+ # When CurrentAttributes are used, this may be a JobWrapper instead of the original job.
95
+ # @return [String] serialized job payload
96
+ def serialize_job(job)
97
+ deserializer.serialize(job)
98
+ end
99
+
74
100
  # @param job [ActiveJob::Base] job
75
101
  # @param key [Symbol] key we want to fetch
76
102
  # @param defaults [Hash]
@@ -81,12 +107,6 @@ module Karafka
81
107
  .karafka_options
82
108
  .fetch(key, defaults.fetch(key))
83
109
  end
84
-
85
- # @param job [ActiveJob::Base] job
86
- # @return [Hash] json representation of the job
87
- def serialize_job(job)
88
- job.serialize
89
- end
90
110
  end
91
111
  end
92
112
  end
@@ -8,10 +8,8 @@ module Karafka
8
8
  # all in the same place
9
9
  class JobOptionsContract < Contracts::Base
10
10
  configure do |config|
11
- config.error_messages = YAML.safe_load(
12
- File.read(
13
- File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
14
- )
11
+ config.error_messages = YAML.safe_load_file(
12
+ File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
15
13
  ).fetch('en').fetch('validations').fetch('job_options')
16
14
  end
17
15
 
@@ -98,8 +98,10 @@ module Karafka
98
98
  PERMISSION_TYPES_MAP
99
99
  ].freeze
100
100
 
101
- private_constant :RESOURCE_TYPES_MAP, :RESOURCE_PATTERNS_TYPE_MAP, :OPERATIONS_MAP,
102
- :PERMISSION_TYPES_MAP, :ALL_MAPS
101
+ private_constant(
102
+ :RESOURCE_TYPES_MAP, :RESOURCE_PATTERNS_TYPE_MAP, :OPERATIONS_MAP, :PERMISSION_TYPES_MAP,
103
+ :ALL_MAPS
104
+ )
103
105
 
104
106
  # Class level APIs that operate on Acl instances and/or return Acl instances.
105
107
  # @note For the sake of consistency all methods from this API return array of Acls
@@ -187,8 +189,10 @@ module Karafka
187
189
  end
188
190
  end
189
191
 
190
- attr_reader :resource_type, :resource_name, :resource_pattern_type, :principal, :host,
191
- :operation, :permission_type
192
+ attr_reader(
193
+ :resource_type, :resource_name, :resource_pattern_type, :principal, :host, :operation,
194
+ :permission_type
195
+ )
192
196
 
193
197
  # Initializes a new Acl instance with specified attributes.
194
198
  #
@@ -71,10 +71,12 @@ module Karafka
71
71
  def synonym? = @synonym.positive?
72
72
 
73
73
  # @return [Hash] hash that we can use to operate with rdkafka
74
- def to_native_hash = {
75
- name: name,
76
- value: value
77
- }.freeze
74
+ def to_native_hash
75
+ {
76
+ name: name,
77
+ value: value
78
+ }.freeze
79
+ end
78
80
  end
79
81
  end
80
82
  end
@@ -40,10 +40,16 @@ module Karafka
40
40
  OPERATIONS_TYPES_MAP.each do |op_name, op_value|
41
41
  # Adds an outgoing operation to a given resource of a given type
42
42
  # Useful since we alter in batches and not one at a time
43
+ #
44
+ # For example, when op_name is :set and op_value is 0:
45
+ # def set(name, value)
46
+ # @operations[0] << Config.new(name: name, value: value.to_s)
47
+ # end
48
+ default_value = op_name == :delete ? ' = nil' : ''
43
49
  class_eval <<~RUBY, __FILE__, __LINE__ + 1
44
50
  # @param name [String] name of the config to alter
45
51
  # @param value [String] value of the config
46
- def #{op_name}(name, value #{op_name == :delete ? ' = nil' : ''})
52
+ def #{op_name}(name, value#{default_value})
47
53
  @operations[#{op_value}] << Config.new(name: name, value: value.to_s)
48
54
  end
49
55
  RUBY
@@ -80,15 +80,13 @@ module Karafka
80
80
  case casted_position
81
81
  # Earliest is not always 0. When compacting/deleting it can be much later, that's why
82
82
  # we fetch the oldest possible offset
83
- when 'earliest'
83
+ # false is treated the same as 'earliest'
84
+ when 'earliest', false
84
85
  LONG_TIME_AGO
85
86
  # Latest will always be the high-watermark offset and we can get it just by getting
86
87
  # a future position
87
88
  when 'latest'
88
89
  Time.now + DAY_IN_SECONDS
89
- # Same as `'earliest'`
90
- when false
91
- LONG_TIME_AGO
92
90
  # Regular offset case
93
91
  else
94
92
  position
@@ -234,17 +232,87 @@ module Karafka
234
232
  end
235
233
  end
236
234
 
235
+ # Triggers a rebalance for the specified consumer group by briefly joining and leaving
236
+ #
237
+ # @param consumer_group_id [String] consumer group id to trigger rebalance for
238
+ #
239
+ # @return [void]
240
+ #
241
+ # @raise [Karafka::Errors::InvalidConfigurationError] when consumer group is not found in
242
+ # routing or has no topics
243
+ #
244
+ # @note This method creates a temporary "fake" consumer that joins the consumer group,
245
+ # triggering a rebalance when it joins and another when it leaves. This should only be
246
+ # used for operational/testing purposes as it causes two rebalances.
247
+ #
248
+ # @note The consumer group does not need to be running for this to work, but if it is,
249
+ # it will experience rebalances.
250
+ #
251
+ # @note The behavior follows the configured rebalance protocol. For cooperative sticky
252
+ # rebalancing or KIP-848 based protocols, there may be no immediate reaction to the
253
+ # rebalance trigger as these protocols allow incremental partition reassignments without
254
+ # stopping all consumers.
255
+ #
256
+ # @note Topics are always detected from the routing configuration. The consumer settings
257
+ # (kafka config) are taken from the first topic in the consumer group to ensure
258
+ # consistency with the actual consumer configuration.
259
+ #
260
+ # @example Trigger rebalance for a consumer group
261
+ # Karafka::Admin::ConsumerGroups.trigger_rebalance('my-group')
262
+ def trigger_rebalance(consumer_group_id)
263
+ consumer_group = Karafka::App.routes.find { |cg| cg.id == consumer_group_id }
264
+
265
+ unless consumer_group
266
+ raise(
267
+ Errors::InvalidConfigurationError,
268
+ "Consumer group '#{consumer_group_id}' not found in routing"
269
+ )
270
+ end
271
+
272
+ topics = consumer_group.topics.map(&:name)
273
+
274
+ if topics.empty?
275
+ raise(
276
+ Errors::InvalidConfigurationError,
277
+ "Consumer group '#{consumer_group_id}' has no topics"
278
+ )
279
+ end
280
+
281
+ # Get the first topic to extract kafka settings
282
+ first_topic = consumer_group.topics.first
283
+
284
+ # Build consumer settings using the consumer group's kafka config from first topic
285
+ # This ensures we use the same settings as the actual consumers
286
+ # Following the same pattern as in Karafka::Connection::Client#build_kafka
287
+ consumer_settings = Setup::AttributesMap.consumer(first_topic.kafka.dup)
288
+ consumer_settings[:'group.id'] = consumer_group.id
289
+ consumer_settings[:'enable.auto.offset.store'] = false
290
+ consumer_settings[:'auto.offset.reset'] ||= first_topic.initial_offset
291
+
292
+ with_consumer(consumer_settings) do |consumer|
293
+ # Subscribe to the topics - this triggers the first rebalance
294
+ consumer.subscribe(*topics)
295
+
296
+ # Wait briefly (100ms) to allow the rebalance to initiate
297
+ # The actual rebalance happens asynchronously, so we just need to give it a moment
298
+ sleep(0.1)
299
+
300
+ # Unsubscribe - this will trigger the second rebalance when the consumer closes
301
+ # The ensure block in with_consumer will handle the unsubscribe and close
302
+ end
303
+ end
304
+
237
305
  # Reads lags and offsets for given topics in the context of consumer groups defined in the
238
306
  # routing
239
307
  #
240
- # @param consumer_groups_with_topics [Hash<String, Array<String>>] hash with consumer
308
+ # @param consumer_groups_with_topics [Hash{String => Array<String>}] hash with consumer
241
309
  # groups names with array of topics to query per consumer group inside
242
310
  # @param active_topics_only [Boolean] if set to false, when we use routing topics, will
243
311
  # select also topics that are marked as inactive in routing
244
312
  #
245
- # @return [Hash<String, Hash<Integer, <Hash<Integer>>>>] hash where the top level keys are
246
- # the consumer groups and values are hashes with topics and inside partitions with lags
247
- # and offsets
313
+ # @return [Hash{String => Hash{Integer => Hash{Integer => Object}}}] hash where the top
314
+ # level keys are the consumer groups and values are hashes with topics and inside
315
+ # partitions with lags and offsets
248
316
  #
249
317
  # @note For topics that do not exist, topic details will be set to an empty hash
250
318
  #
@@ -257,19 +325,19 @@ module Karafka
257
325
  # We first fetch all the topics with partitions count that exist in the cluster so we
258
326
  # do not query for topics that do not exist and so we can get partitions count for all
259
327
  # the topics we may need. The non-existent and not consumed will be filled at the end
260
- existing_topics = cluster_info.topics.map do |topic|
328
+ existing_topics = cluster_info.topics.to_h do |topic|
261
329
  [topic[:topic_name], topic[:partition_count]]
262
- end.to_h.freeze
330
+ end.freeze
263
331
 
264
332
  # If no expected CGs, we use all from routing that have active topics
265
333
  if consumer_groups_with_topics.empty?
266
- consumer_groups_with_topics = Karafka::App.routes.map do |cg|
334
+ consumer_groups_with_topics = Karafka::App.routes.to_h do |cg|
267
335
  cg_topics = cg.topics.select do |cg_topic|
268
336
  active_topics_only ? cg_topic.active? : true
269
337
  end
270
338
 
271
339
  [cg.id, cg_topics.map(&:name)]
272
- end.to_h
340
+ end
273
341
  end
274
342
 
275
343
  # We make a copy because we will remove once with non-existing topics
@@ -36,7 +36,7 @@ module Karafka
36
36
 
37
37
  # Build the requested range - since first element is on the start offset we need to
38
38
  # subtract one from requested count to end up with expected number of elements
39
- requested_range = (start_offset..start_offset + (count - 1))
39
+ requested_range = (start_offset..(start_offset + count - 1))
40
40
  # Establish theoretical available range. Note, that this does not handle cases related
41
41
  # to log retention or compaction
42
42
  available_range = (low_offset..(high_offset - 1))
@@ -75,7 +75,7 @@ module Karafka
75
75
  # Use topic from routes if we can match it or create a dummy one
76
76
  # Dummy one is used in case we cannot match the topic with routes. This can happen
77
77
  # when admin API is used to read topics that are not part of the routing
78
- topic = ::Karafka::Routing::Router.find_or_initialize_by_name(name)
78
+ topic = Karafka::Routing::Router.find_or_initialize_by_name(name)
79
79
 
80
80
  messages.map! do |message|
81
81
  Messages::Builders::Message.call(
@@ -139,15 +139,49 @@ module Karafka
139
139
  end
140
140
  end
141
141
 
142
- # Fetches the watermark offsets for a given topic partition
142
+ # Fetches the watermark offsets for a given topic partition or multiple topics and
143
+ # partitions
143
144
  #
144
- # @param name [String, Symbol] topic name
145
- # @param partition [Integer] partition
146
- # @return [Array<Integer, Integer>] low watermark offset and high watermark offset
147
- def read_watermark_offsets(name, partition)
145
+ # @param name_or_hash [String, Symbol, Hash] topic name or hash with topics and partitions
146
+ # @param partition [Integer, nil] partition number
147
+ # (required when first param is topic name)
148
+ #
149
+ # @return [Array<Integer, Integer>, Hash] when querying single partition returns array with
150
+ # low and high watermark offsets, when querying multiple returns nested hash
151
+ #
152
+ # @example Query single partition
153
+ # Karafka::Admin::Topics.read_watermark_offsets('events', 0)
154
+ # # => [0, 100]
155
+ #
156
+ # @example Query specific partitions across multiple topics
157
+ # Karafka::Admin::Topics.read_watermark_offsets(
158
+ # { 'events' => [0, 1], 'logs' => [0] }
159
+ # )
160
+ # # => {
161
+ # # 'events' => {
162
+ # # 0 => [0, 100],
163
+ # # 1 => [0, 150]
164
+ # # },
165
+ # # 'logs' => {
166
+ # # 0 => [0, 50]
167
+ # # }
168
+ # # }
169
+ def read_watermark_offsets(name_or_hash, partition = nil)
170
+ # Normalize input to hash format
171
+ topics_with_partitions = partition ? { name_or_hash => [partition] } : name_or_hash
172
+
173
+ result = Hash.new { |h, k| h[k] = {} }
174
+
148
175
  with_consumer do |consumer|
149
- consumer.query_watermark_offsets(name, partition)
176
+ topics_with_partitions.each do |topic, partitions|
177
+ partitions.each do |partition_id|
178
+ result[topic][partition_id] = consumer.query_watermark_offsets(topic, partition_id)
179
+ end
180
+ end
150
181
  end
182
+
183
+ # Return single array for single partition query, hash for multiple
184
+ partition ? result.dig(name_or_hash, partition) : result
151
185
  end
152
186
 
153
187
  # Returns basic topic metadata
@@ -184,7 +218,7 @@ module Karafka
184
218
  # @return [Integer] expected offset
185
219
  def resolve_offset(consumer, name, partition, offset)
186
220
  if offset.is_a?(Time)
187
- tpl = ::Rdkafka::Consumer::TopicPartitionList.new
221
+ tpl = Rdkafka::Consumer::TopicPartitionList.new
188
222
  tpl.add_topic_and_partitions_with_offsets(
189
223
  name, partition => offset
190
224
  )
data/lib/karafka/admin.rb CHANGED
@@ -59,11 +59,11 @@ module Karafka
59
59
  Topics.create_partitions(name, partitions)
60
60
  end
61
61
 
62
- # @param name [String, Symbol] topic name
63
- # @param partition [Integer] partition
62
+ # @param name_or_hash [String, Symbol, Hash] topic name or hash with topics and partitions
63
+ # @param partition [Integer, nil] partition (nil when using hash format)
64
64
  # @see Topics.read_watermark_offsets
65
- def read_watermark_offsets(name, partition)
66
- Topics.read_watermark_offsets(name, partition)
65
+ def read_watermark_offsets(name_or_hash, partition = nil)
66
+ Topics.read_watermark_offsets(name_or_hash, partition)
67
67
  end
68
68
 
69
69
  # @param topic_name [String] name of the topic we're interested in
@@ -113,15 +113,24 @@ module Karafka
113
113
  ConsumerGroups.delete(consumer_group_id)
114
114
  end
115
115
 
116
+ # Triggers a rebalance for the specified consumer group
117
+ #
118
+ # @param consumer_group_id [String] consumer group id to trigger rebalance for
119
+ # @see ConsumerGroups.trigger_rebalance
120
+ # @note This API should be used only for development.
121
+ def trigger_rebalance(consumer_group_id)
122
+ ConsumerGroups.trigger_rebalance(consumer_group_id)
123
+ end
124
+
116
125
  # Reads lags and offsets for given topics in the context of consumer groups defined in the
117
126
  # routing
118
- # @param consumer_groups_with_topics [Hash<String, Array<String>>] hash with consumer groups
119
- # names with array of topics to query per consumer group inside
127
+ # @param consumer_groups_with_topics [Hash{String => Array<String>}] hash with consumer
128
+ # groups names with array of topics to query per consumer group inside
120
129
  # @param active_topics_only [Boolean] if set to false, when we use routing topics, will
121
130
  # select also topics that are marked as inactive in routing
122
- # @return [Hash<String, Hash<Integer, <Hash<Integer>>>>] hash where the top level keys are
123
- # the consumer groups and values are hashes with topics and inside partitions with lags
124
- # and offsets
131
+ # @return [Hash{String => Hash{Integer => Hash{Integer => Object}}}] hash where the top
132
+ # level keys are the consumer groups and values are hashes with topics and inside
133
+ # partitions with lags and offsets
125
134
  # @see ConsumerGroups.read_lags_with_offsets
126
135
  def read_lags_with_offsets(consumer_groups_with_topics = {}, active_topics_only: true)
127
136
  ConsumerGroups.read_lags_with_offsets(
@@ -149,7 +158,7 @@ module Karafka
149
158
  bind_oauth(bind_id, consumer)
150
159
 
151
160
  consumer.start
152
- proxy = ::Karafka::Connection::Proxy.new(consumer)
161
+ proxy = Karafka::Connection::Proxy.new(consumer)
153
162
  yield(proxy)
154
163
  ensure
155
164
  # Always unsubscribe consumer just to be sure, that no metadata requests are running
@@ -179,7 +188,7 @@ module Karafka
179
188
  bind_oauth(bind_id, admin)
180
189
 
181
190
  admin.start
182
- proxy = ::Karafka::Connection::Proxy.new(admin)
191
+ proxy = Karafka::Connection::Proxy.new(admin)
183
192
  yield(proxy)
184
193
  ensure
185
194
  admin&.close
@@ -202,7 +211,7 @@ module Karafka
202
211
  # @param instance [Rdkafka::Consumer, Rdkafka::Admin] rdkafka instance to be used to set
203
212
  # appropriate oauth token when needed
204
213
  def bind_oauth(id, instance)
205
- ::Karafka::Core::Instrumentation.oauthbearer_token_refresh_callbacks.add(
214
+ Karafka::Core::Instrumentation.oauthbearer_token_refresh_callbacks.add(
206
215
  id,
207
216
  Instrumentation::Callbacks::OauthbearerTokenRefresh.new(
208
217
  instance
@@ -215,7 +224,7 @@ module Karafka
215
224
  # @param id [String, Symbol] unique (for the lifetime of instance) id that we use for
216
225
  # callback referencing
217
226
  def unbind_oauth(id)
218
- ::Karafka::Core::Instrumentation.oauthbearer_token_refresh_callbacks.delete(id)
227
+ Karafka::Core::Instrumentation.oauthbearer_token_refresh_callbacks.delete(id)
219
228
  end
220
229
 
221
230
  # There are some cases where rdkafka admin operations finish successfully but without the
@@ -260,7 +269,7 @@ module Karafka
260
269
  # consumer group or do something similar
261
270
  .merge!(settings)
262
271
  .then { |config| Karafka::Setup::AttributesMap.public_send(type, config) }
263
- .then { |config| ::Rdkafka::Config.new(config) }
272
+ .then { |config| Rdkafka::Config.new(config) }
264
273
  end
265
274
  end
266
275
  end
data/lib/karafka/app.rb CHANGED
@@ -52,7 +52,7 @@ module Karafka
52
52
 
53
53
  # Returns current assignments of this process. Both topics and partitions
54
54
  #
55
- # @return [Hash<Karafka::Routing::Topic, Array<Integer>>]
55
+ # @return [Hash{Karafka::Routing::Topic => Array<Integer>}]
56
56
  def assignments
57
57
  Instrumentation::AssignmentsTracker.instance.current
58
58
  end
@@ -102,8 +102,8 @@ module Karafka
102
102
  #
103
103
  # @param contexts [String] librdkafka low level debug contexts for granular debugging
104
104
  def debug!(contexts = 'all')
105
- logger.level = ::Logger::DEBUG
106
- producer.config.logger.level = ::Logger::DEBUG
105
+ logger.level = Logger::DEBUG
106
+ producer.config.logger.level = Logger::DEBUG
107
107
 
108
108
  config.kafka[:debug] = contexts
109
109
  producer.config.kafka[:debug] = contexts
@@ -5,7 +5,7 @@ module Karafka
5
5
  # Base consumer from which all Karafka consumers should inherit
6
6
  class BaseConsumer
7
7
  # Allow for consumer instance tagging for instrumentation
8
- include ::Karafka::Core::Taggable
8
+ include Karafka::Core::Taggable
9
9
  include Helpers::ConfigImporter.new(
10
10
  monitor: %i[monitor]
11
11
  )
@@ -14,8 +14,9 @@ module Karafka
14
14
 
15
15
  def_delegators :@coordinator, :topic, :partition, :eofed?, :seek_offset, :seek_offset=
16
16
 
17
- def_delegators :producer, :produce_async, :produce_sync, :produce_many_async,
18
- :produce_many_sync
17
+ def_delegators(
18
+ :producer, :produce_async, :produce_sync, :produce_many_async, :produce_many_sync
19
+ )
19
20
 
20
21
  def_delegators :messages, :each
21
22
 
@@ -81,9 +82,8 @@ module Karafka
81
82
  # @private
82
83
  #
83
84
  # @param action [Symbol]
84
- # @param block [Proc]
85
- def on_wrap(action, &block)
86
- handle_wrap(action, &block)
85
+ def on_wrap(action, &)
86
+ handle_wrap(action, &)
87
87
  rescue StandardError => e
88
88
  monitor.instrument(
89
89
  'error.occurred',
@@ -68,11 +68,11 @@ module Karafka
68
68
 
69
69
  # However when it is unavailable, we still want to be able to run help command
70
70
  # and install command as they don't require configured app itself to run
71
- return if %w[-h install].any? { |cmd| cmd == ARGV[0] }
71
+ return if %w[-h install].any?(ARGV[0])
72
72
 
73
73
  # All other commands except help and install do require an existing boot file if it was
74
74
  # declared
75
- raise ::Karafka::Errors::MissingBootFileError, ::Karafka.boot_file
75
+ raise Karafka::Errors::MissingBootFileError, Karafka.boot_file
76
76
  end
77
77
 
78
78
  # Allows to set options for Thor cli
@@ -28,7 +28,7 @@ module Karafka
28
28
  def call
29
29
  Info.new.call
30
30
 
31
- command = ::Karafka.rails? ? self.class.rails_console : self.class.console
31
+ command = Karafka.rails? ? self.class.rails_console : self.class.console
32
32
 
33
33
  exec "KARAFKA_CONSOLE=true #{command}"
34
34
  end
@@ -5,12 +5,10 @@ module Karafka
5
5
  # CLI related contracts
6
6
  module Contracts
7
7
  # Contract for validating correctness of the server cli command options.
8
- class Server < ::Karafka::Contracts::Base
8
+ class Server < Karafka::Contracts::Base
9
9
  configure do |config|
10
- config.error_messages = YAML.safe_load(
11
- File.read(
12
- File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
13
- )
10
+ config.error_messages = YAML.safe_load_file(
11
+ File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
14
12
  ).fetch('en').fetch('validations').fetch('cli').fetch('server')
15
13
  end
16
14