karafka 2.5.2 → 2.5.4.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (201) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +16 -0
  3. data/config/locales/errors.yml +14 -0
  4. data/karafka.gemspec +15 -4
  5. data/lib/active_job/queue_adapters/karafka_adapter.rb +2 -2
  6. data/lib/karafka/active_job/consumer.rb +2 -2
  7. data/lib/karafka/active_job/current_attributes.rb +2 -2
  8. data/lib/karafka/active_job/deserializer.rb +1 -1
  9. data/lib/karafka/active_job/dispatcher.rb +2 -2
  10. data/lib/karafka/admin/configs/resource.rb +7 -1
  11. data/lib/karafka/admin/consumer_groups.rb +6 -8
  12. data/lib/karafka/admin/contracts/replication.rb +149 -0
  13. data/lib/karafka/admin/replication.rb +462 -0
  14. data/lib/karafka/admin/topics.rb +5 -4
  15. data/lib/karafka/admin.rb +57 -12
  16. data/lib/karafka/app.rb +3 -3
  17. data/lib/karafka/base_consumer.rb +1 -1
  18. data/lib/karafka/cli/base.rb +1 -1
  19. data/lib/karafka/cli/console.rb +1 -1
  20. data/lib/karafka/cli/contracts/server.rb +1 -1
  21. data/lib/karafka/cli/help.rb +1 -1
  22. data/lib/karafka/cli/install.rb +2 -1
  23. data/lib/karafka/cli/server.rb +1 -1
  24. data/lib/karafka/cli/swarm.rb +1 -1
  25. data/lib/karafka/connection/client.rb +19 -18
  26. data/lib/karafka/connection/manager.rb +1 -0
  27. data/lib/karafka/connection/proxy.rb +1 -1
  28. data/lib/karafka/connection/rebalance_manager.rb +1 -1
  29. data/lib/karafka/connection/status.rb +1 -0
  30. data/lib/karafka/constraints.rb +1 -1
  31. data/lib/karafka/contracts/base.rb +1 -1
  32. data/lib/karafka/deserializers/payload.rb +1 -1
  33. data/lib/karafka/helpers/async.rb +1 -1
  34. data/lib/karafka/helpers/config_importer.rb +3 -3
  35. data/lib/karafka/helpers/multi_delegator.rb +3 -0
  36. data/lib/karafka/instrumentation/assignments_tracker.rb +2 -1
  37. data/lib/karafka/instrumentation/callbacks/error.rb +2 -2
  38. data/lib/karafka/instrumentation/callbacks/statistics.rb +3 -3
  39. data/lib/karafka/instrumentation/logger.rb +6 -6
  40. data/lib/karafka/instrumentation/logger_listener.rb +0 -2
  41. data/lib/karafka/instrumentation/monitor.rb +2 -2
  42. data/lib/karafka/instrumentation/vendors/appsignal/base.rb +1 -1
  43. data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb +4 -0
  44. data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +32 -16
  45. data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +2 -2
  46. data/lib/karafka/instrumentation/vendors/kubernetes/base_listener.rb +1 -1
  47. data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +3 -15
  48. data/lib/karafka/licenser.rb +1 -1
  49. data/lib/karafka/messages/builders/batch_metadata.rb +1 -1
  50. data/lib/karafka/messages/messages.rb +32 -0
  51. data/lib/karafka/pro/active_job/consumer.rb +2 -2
  52. data/lib/karafka/pro/active_job/dispatcher.rb +3 -3
  53. data/lib/karafka/pro/cleaner/messages/messages.rb +1 -1
  54. data/lib/karafka/pro/cleaner.rb +3 -3
  55. data/lib/karafka/pro/cli/contracts/server.rb +1 -1
  56. data/lib/karafka/pro/cli/parallel_segments/base.rb +4 -3
  57. data/lib/karafka/pro/cli/parallel_segments/collapse.rb +1 -1
  58. data/lib/karafka/pro/cli/parallel_segments/distribute.rb +1 -1
  59. data/lib/karafka/pro/cli/parallel_segments.rb +1 -1
  60. data/lib/karafka/pro/connection/manager.rb +1 -2
  61. data/lib/karafka/pro/connection/multiplexing/listener.rb +1 -0
  62. data/lib/karafka/pro/contracts/base.rb +1 -1
  63. data/lib/karafka/pro/encryption/cipher.rb +3 -2
  64. data/lib/karafka/pro/encryption/contracts/config.rb +1 -1
  65. data/lib/karafka/pro/encryption/messages/parser.rb +1 -1
  66. data/lib/karafka/pro/encryption/setup/config.rb +1 -1
  67. data/lib/karafka/pro/iterator/tpl_builder.rb +1 -1
  68. data/lib/karafka/pro/iterator.rb +1 -1
  69. data/lib/karafka/pro/loader.rb +1 -1
  70. data/lib/karafka/pro/processing/coordinator.rb +1 -1
  71. data/lib/karafka/pro/processing/filters/base.rb +1 -0
  72. data/lib/karafka/pro/processing/filters/delayer.rb +1 -1
  73. data/lib/karafka/pro/processing/filters/expirer.rb +1 -1
  74. data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb +1 -1
  75. data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +1 -1
  76. data/lib/karafka/pro/processing/jobs/eofed_non_blocking.rb +1 -1
  77. data/lib/karafka/pro/processing/jobs/periodic.rb +1 -1
  78. data/lib/karafka/pro/processing/jobs/revoked_non_blocking.rb +1 -1
  79. data/lib/karafka/pro/processing/jobs_builder.rb +1 -1
  80. data/lib/karafka/pro/processing/jobs_queue.rb +0 -2
  81. data/lib/karafka/pro/processing/offset_metadata/fetcher.rb +1 -0
  82. data/lib/karafka/pro/processing/partitioner.rb +1 -1
  83. data/lib/karafka/pro/processing/strategies/base.rb +1 -1
  84. data/lib/karafka/pro/processing/strategies/default.rb +2 -2
  85. data/lib/karafka/pro/processing/strategies/dlq/default.rb +1 -1
  86. data/lib/karafka/pro/processing/strategies/vp/default.rb +1 -1
  87. data/lib/karafka/pro/processing/strategy_selector.rb +1 -0
  88. data/lib/karafka/pro/processing/virtual_partitions/distributors/balanced.rb +4 -2
  89. data/lib/karafka/pro/processing/virtual_partitions/distributors/consistent.rb +4 -2
  90. data/lib/karafka/pro/recurring_tasks/consumer.rb +3 -2
  91. data/lib/karafka/pro/recurring_tasks/contracts/config.rb +2 -2
  92. data/lib/karafka/pro/recurring_tasks/contracts/task.rb +1 -1
  93. data/lib/karafka/pro/recurring_tasks/deserializer.rb +1 -1
  94. data/lib/karafka/pro/recurring_tasks/dispatcher.rb +1 -1
  95. data/lib/karafka/pro/recurring_tasks/executor.rb +2 -1
  96. data/lib/karafka/pro/recurring_tasks/schedule.rb +5 -2
  97. data/lib/karafka/pro/recurring_tasks/serializer.rb +6 -5
  98. data/lib/karafka/pro/recurring_tasks/setup/config.rb +2 -2
  99. data/lib/karafka/pro/recurring_tasks/task.rb +1 -1
  100. data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb +3 -0
  101. data/lib/karafka/pro/routing/features/multiplexing/subscription_groups_builder.rb +1 -1
  102. data/lib/karafka/pro/routing/features/multiplexing.rb +5 -5
  103. data/lib/karafka/pro/routing/features/offset_metadata.rb +4 -4
  104. data/lib/karafka/pro/routing/features/parallel_segments/builder.rb +1 -1
  105. data/lib/karafka/pro/routing/features/patterns/patterns.rb +1 -1
  106. data/lib/karafka/pro/routing/features/periodic_job/topic.rb +1 -1
  107. data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb +1 -1
  108. data/lib/karafka/pro/routing/features/swarm.rb +1 -1
  109. data/lib/karafka/pro/routing/features/throttling/topic.rb +3 -1
  110. data/lib/karafka/pro/scheduled_messages/consumer.rb +1 -1
  111. data/lib/karafka/pro/scheduled_messages/contracts/config.rb +2 -2
  112. data/lib/karafka/pro/scheduled_messages/contracts/message.rb +1 -1
  113. data/lib/karafka/pro/scheduled_messages/daily_buffer.rb +3 -2
  114. data/lib/karafka/pro/scheduled_messages/day.rb +1 -0
  115. data/lib/karafka/pro/scheduled_messages/deserializers/headers.rb +1 -1
  116. data/lib/karafka/pro/scheduled_messages/deserializers/payload.rb +1 -1
  117. data/lib/karafka/pro/scheduled_messages/max_epoch.rb +1 -0
  118. data/lib/karafka/pro/scheduled_messages/proxy.rb +1 -1
  119. data/lib/karafka/pro/scheduled_messages/serializer.rb +3 -3
  120. data/lib/karafka/pro/scheduled_messages/setup/config.rb +2 -2
  121. data/lib/karafka/pro/scheduled_messages/state.rb +1 -0
  122. data/lib/karafka/pro/scheduled_messages/tracker.rb +1 -0
  123. data/lib/karafka/process.rb +4 -4
  124. data/lib/karafka/processing/executor.rb +1 -1
  125. data/lib/karafka/processing/inline_insights/tracker.rb +1 -0
  126. data/lib/karafka/processing/jobs_queue.rb +1 -1
  127. data/lib/karafka/processing/result.rb +1 -0
  128. data/lib/karafka/processing/strategies/dlq.rb +1 -1
  129. data/lib/karafka/processing/strategy_selector.rb +1 -0
  130. data/lib/karafka/routing/activity_manager.rb +1 -0
  131. data/lib/karafka/routing/builder.rb +3 -1
  132. data/lib/karafka/routing/consumer_group.rb +19 -1
  133. data/lib/karafka/routing/contracts/consumer_group.rb +3 -2
  134. data/lib/karafka/routing/contracts/topic.rb +5 -2
  135. data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +1 -1
  136. data/lib/karafka/routing/features/declaratives/topic.rb +5 -2
  137. data/lib/karafka/routing/features/deserializers/topic.rb +3 -3
  138. data/lib/karafka/routing/features/inline_insights.rb +5 -5
  139. data/lib/karafka/routing/router.rb +1 -1
  140. data/lib/karafka/routing/subscription_group.rb +2 -2
  141. data/lib/karafka/routing/subscription_groups_builder.rb +18 -2
  142. data/lib/karafka/routing/topic.rb +3 -3
  143. data/lib/karafka/server.rb +1 -1
  144. data/lib/karafka/setup/attributes_map.rb +4 -2
  145. data/lib/karafka/setup/config.rb +21 -10
  146. data/lib/karafka/setup/config_proxy.rb +209 -0
  147. data/lib/karafka/setup/contracts/config.rb +1 -1
  148. data/lib/karafka/swarm/liveness_listener.rb +1 -0
  149. data/lib/karafka/swarm/manager.rb +7 -6
  150. data/lib/karafka/swarm/node.rb +1 -1
  151. data/lib/karafka/swarm/supervisor.rb +1 -0
  152. data/lib/karafka/time_trackers/base.rb +1 -1
  153. data/lib/karafka/version.rb +1 -1
  154. data/lib/karafka.rb +2 -3
  155. metadata +8 -65
  156. data/.coditsu/ci.yml +0 -3
  157. data/.console_irbrc +0 -11
  158. data/.github/CODEOWNERS +0 -3
  159. data/.github/FUNDING.yml +0 -1
  160. data/.github/ISSUE_TEMPLATE/bug_report.md +0 -43
  161. data/.github/ISSUE_TEMPLATE/feature_request.md +0 -20
  162. data/.github/workflows/ci_linux_ubuntu_x86_64_gnu.yml +0 -278
  163. data/.github/workflows/ci_macos_arm64.yml +0 -151
  164. data/.github/workflows/push.yml +0 -35
  165. data/.github/workflows/trigger-wiki-refresh.yml +0 -30
  166. data/.github/workflows/verify-action-pins.yml +0 -16
  167. data/.gitignore +0 -69
  168. data/.rspec +0 -7
  169. data/.ruby-gemset +0 -1
  170. data/.ruby-version +0 -1
  171. data/CODE_OF_CONDUCT.md +0 -46
  172. data/CONTRIBUTING.md +0 -32
  173. data/Gemfile +0 -28
  174. data/Gemfile.lock +0 -173
  175. data/Rakefile +0 -4
  176. data/SECURITY.md +0 -23
  177. data/bin/benchmarks +0 -99
  178. data/bin/clean_kafka +0 -43
  179. data/bin/create_token +0 -22
  180. data/bin/integrations +0 -341
  181. data/bin/record_rss +0 -50
  182. data/bin/rspecs +0 -26
  183. data/bin/scenario +0 -29
  184. data/bin/stress_many +0 -13
  185. data/bin/stress_one +0 -13
  186. data/bin/verify_kafka_warnings +0 -36
  187. data/bin/verify_license_integrity +0 -37
  188. data/bin/verify_topics_naming +0 -27
  189. data/bin/wait_for_kafka +0 -24
  190. data/docker-compose.yml +0 -25
  191. data/examples/payloads/avro/.gitkeep +0 -0
  192. data/examples/payloads/json/sample_set_01/enrollment_event.json +0 -579
  193. data/examples/payloads/json/sample_set_01/ingestion_event.json +0 -30
  194. data/examples/payloads/json/sample_set_01/transaction_event.json +0 -17
  195. data/examples/payloads/json/sample_set_01/user_event.json +0 -11
  196. data/examples/payloads/json/sample_set_02/download.json +0 -191
  197. data/examples/payloads/json/sample_set_03/event_type_1.json +0 -18
  198. data/examples/payloads/json/sample_set_03/event_type_2.json +0 -263
  199. data/examples/payloads/json/sample_set_03/event_type_3.json +0 -41
  200. data/log/.gitkeep +0 -0
  201. data/renovate.json +0 -21
@@ -14,7 +14,7 @@ module Karafka
14
14
  #
15
15
  # It contains slightly better revocation warranties than the regular blocking consumer as
16
16
  # it can stop processing batch of jobs in the middle after the revocation.
17
- class Consumer < ::Karafka::ActiveJob::Consumer
17
+ class Consumer < Karafka::ActiveJob::Consumer
18
18
  # Runs ActiveJob jobs processing and handles lrj if needed
19
19
  def consume
20
20
  messages.each(clean: true) do |message|
@@ -25,7 +25,7 @@ module Karafka
25
25
  # We cannot early stop when running virtual partitions because the intermediate state
26
26
  # would force us not to commit the offsets. This would cause extensive
27
27
  # double-processing
28
- break if Karafka::App.stopping? && !topic.virtual_partitions?
28
+ break if ::Karafka::App.stopping? && !topic.virtual_partitions?
29
29
 
30
30
  consume_job(message)
31
31
 
@@ -10,7 +10,7 @@ module Karafka
10
10
  # Pro dispatcher that sends the ActiveJob job to a proper topic based on the queue name
11
11
  # and that allows to inject additional options into the producer, effectively allowing for a
12
12
  # much better and more granular control over the dispatch and consumption process.
13
- class Dispatcher < ::Karafka::ActiveJob::Dispatcher
13
+ class Dispatcher < Karafka::ActiveJob::Dispatcher
14
14
  include Helpers::ConfigImporter.new(
15
15
  deserializer: %i[internal active_job deserializer]
16
16
  )
@@ -31,7 +31,7 @@ module Karafka
31
31
  # Allows for setting a callable producer since at the moment of defining the class,
32
32
  # variants may not be available
33
33
  #
34
- # We do not initialize it with `-> { ::Karafka.producer }` so we do not have to call it
34
+ # We do not initialize it with `-> { Karafka.producer }` so we do not have to call it
35
35
  # each time for the defaults to preserve CPU cycles.
36
36
  #
37
37
  # We also do **not** cache the execution of this producer lambda because we want to
@@ -122,7 +122,7 @@ module Karafka
122
122
  def producer(job)
123
123
  dynamic_producer = fetch_option(job, :producer, DEFAULTS)
124
124
 
125
- dynamic_producer ? dynamic_producer.call(job) : ::Karafka.producer
125
+ dynamic_producer ? dynamic_producer.call(job) : Karafka.producer
126
126
  end
127
127
 
128
128
  # @param job [ActiveJob::Base] job instance
@@ -13,7 +13,7 @@ module Karafka
13
13
  # This module is prepended to Karafka::Messages::Messages to add cleaning functionality.
14
14
  # The implementation calls super() to maintain compatibility with other libraries that
15
15
  # also prepend modules to modify the #each method (e.g., DataDog tracing).
16
- # See: https://github.com/DataDog/dd-trace-rb/issues/4867
16
+ # @see https://github.com/DataDog/dd-trace-rb/issues/4867
17
17
  module Messages
18
18
  # @param clean [Boolean] do we want to clean each message after we're done working with
19
19
  # it.
@@ -19,9 +19,9 @@ module Karafka
19
19
  class << self
20
20
  # @param _config [Karafka::Core::Configurable::Node] root node config
21
21
  def pre_setup(_config)
22
- ::Karafka::Messages::Message.prepend(Messages::Message)
23
- ::Karafka::Messages::Metadata.prepend(Messages::Metadata)
24
- ::Karafka::Messages::Messages.prepend(Messages::Messages)
22
+ Karafka::Messages::Message.prepend(Messages::Message)
23
+ Karafka::Messages::Metadata.prepend(Messages::Metadata)
24
+ Karafka::Messages::Messages.prepend(Messages::Messages)
25
25
  end
26
26
 
27
27
  # @param _config [Karafka::Core::Configurable::Node] root node config
@@ -10,7 +10,7 @@ module Karafka
10
10
  module Contracts
11
11
  # Contract for validating correctness of the server cli command options.
12
12
  # It differs slightly from the OSS one because it is aware of the routing patterns
13
- class Server < ::Karafka::Cli::Contracts::Server
13
+ class Server < Karafka::Cli::Contracts::Server
14
14
  configure do |config|
15
15
  config.error_messages = YAML.safe_load_file(
16
16
  File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
@@ -12,6 +12,7 @@ module Karafka
12
12
  include Helpers::Colorize
13
13
 
14
14
  # @param options [Hash] cli flags options
15
+ # @option options [Array<String>] :groups consumer group names to work with
15
16
  def initialize(options)
16
17
  @options = options
17
18
  end
@@ -23,12 +24,12 @@ module Karafka
23
24
 
24
25
  # Returns consumer groups for parallel segments with which we should be working
25
26
  #
26
- # @return [Hash<String, Array<Karafka::Routing::ConsumerGroup>>] hash with all parallel
27
+ # @return [Hash{String => Array<Karafka::Routing::ConsumerGroup>}] hash with all parallel
27
28
  # consumer groups as values and names of segments origin consumer group as the key.
28
29
  def applicable_groups
29
30
  requested_groups = options[:groups].dup || []
30
31
 
31
- workable_groups = ::Karafka::App
32
+ workable_groups = Karafka::App
32
33
  .routes
33
34
  .select(&:parallel_segments?)
34
35
  .group_by(&:segment_origin)
@@ -46,7 +47,7 @@ module Karafka
46
47
  applicable_groups[requested_group] = workable_group
47
48
  else
48
49
  raise(
49
- ::Karafka::Errors::ConsumerGroupNotFoundError,
50
+ Karafka::Errors::ConsumerGroupNotFoundError,
50
51
  "Consumer group #{requested_group} was not found"
51
52
  )
52
53
  end
@@ -132,7 +132,7 @@ module Karafka
132
132
  return unless inconclusive
133
133
 
134
134
  raise(
135
- ::Karafka::Errors::CommandValidationError,
135
+ Karafka::Errors::CommandValidationError,
136
136
  "Parallel segments for #{red(segment_origin)} have #{red('inconclusive')} offsets"
137
137
  )
138
138
  end
@@ -91,7 +91,7 @@ module Karafka
91
91
  next unless offset.to_i.positive?
92
92
 
93
93
  raise(
94
- ::Karafka::Errors::CommandValidationError,
94
+ Karafka::Errors::CommandValidationError,
95
95
  "Parallel segment #{red(cg_name)} already has offset #{red(offset)} " \
96
96
  "set for #{red("#{topic_name}##{partition_id}")}"
97
97
  )
@@ -51,7 +51,7 @@ module Karafka
51
51
  Collapse.new(options).call
52
52
  Distribute.new(options).call
53
53
  else
54
- raise ::ArgumentError, "Invalid topics action: #{action}"
54
+ raise ArgumentError, "Invalid topics action: #{action}"
55
55
  end
56
56
  end
57
57
  end
@@ -184,8 +184,7 @@ module Karafka
184
184
  .assignments
185
185
  .select { |_, partitions| partitions.size > 1 }
186
186
  .keys
187
- .map(&:subscription_group)
188
- .map(&:name)
187
+ .map { |sg| sg.subscription_group.name }
189
188
  .uniq
190
189
 
191
190
  # Select connections for scaling up
@@ -12,6 +12,7 @@ module Karafka
12
12
  # Listener used to connect listeners manager to the lifecycle events that are significant
13
13
  # to its operations
14
14
  class Listener
15
+ # Initializes the multiplexing listener with the connection manager
15
16
  def initialize
16
17
  @manager = App.config.internal.connection.manager
17
18
  end
@@ -8,7 +8,7 @@ module Karafka
8
8
  # Pro contracts that aim to replace or complement the general framework contracts
9
9
  module Contracts
10
10
  # Base for all the Pro contracts
11
- class Base < ::Karafka::Contracts::Base
11
+ class Base < Karafka::Contracts::Base
12
12
  end
13
13
  end
14
14
  end
@@ -12,6 +12,7 @@ module Karafka
12
12
  encryption: %i[encryption]
13
13
  )
14
14
 
15
+ # Initializes the cipher with empty private keys cache
15
16
  def initialize
16
17
  @private_pems = {}
17
18
  end
@@ -35,7 +36,7 @@ module Karafka
35
36
 
36
37
  # @return [::OpenSSL::PKey::RSA] rsa public key
37
38
  def public_pem
38
- @public_pem ||= ::OpenSSL::PKey::RSA.new(encryption.public_key)
39
+ @public_pem ||= OpenSSL::PKey::RSA.new(encryption.public_key)
39
40
  end
40
41
 
41
42
  # @param version [String] version for which we want to get the rsa key
@@ -46,7 +47,7 @@ module Karafka
46
47
  key_string = encryption.private_keys[version]
47
48
  key_string || raise(Errors::PrivateKeyNotFoundError, version)
48
49
 
49
- @private_pems[version] = ::OpenSSL::PKey::RSA.new(key_string)
50
+ @private_pems[version] = OpenSSL::PKey::RSA.new(key_string)
50
51
  end
51
52
  end
52
53
  end
@@ -9,7 +9,7 @@ module Karafka
9
9
  # Encryption related contracts
10
10
  module Contracts
11
11
  # Makes sure, all the expected config is defined as it should be
12
- class Config < ::Karafka::Contracts::Base
12
+ class Config < Karafka::Contracts::Base
13
13
  configure do |config|
14
14
  config.error_messages = YAML.safe_load_file(
15
15
  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
@@ -11,7 +11,7 @@ module Karafka
11
11
  # @note There may be a case where someone decides not to encrypt data and we start getting
12
12
  # unencrypted payloads. That is why we always rely on message headers for encryption
13
13
  # indication.
14
- class Parser < ::Karafka::Messages::Parser
14
+ class Parser < Karafka::Messages::Parser
15
15
  include Helpers::ConfigImporter.new(
16
16
  cipher: %i[encryption cipher],
17
17
  active: %i[encryption active],
@@ -10,7 +10,7 @@ module Karafka
10
10
  module Setup
11
11
  # Config for encryption
12
12
  class Config
13
- extend ::Karafka::Core::Configurable
13
+ extend Karafka::Core::Configurable
14
14
 
15
15
  # Should this feature be in use
16
16
  setting(:active, default: false)
@@ -22,7 +22,7 @@ module Karafka
22
22
  # @param consumer [::Rdkafka::Consumer] consumer instance needed to talk with Kafka
23
23
  # @param expanded_topics [Hash] hash with expanded and normalized topics data
24
24
  def initialize(consumer, expanded_topics)
25
- @consumer = ::Karafka::Connection::Proxy.new(consumer)
25
+ @consumer = Karafka::Connection::Proxy.new(consumer)
26
26
  @expanded_topics = expanded_topics
27
27
  @mapped_topics = Hash.new { |h, k| h[k] = {} }
28
28
  end
@@ -44,7 +44,7 @@ module Karafka
44
44
  @topics_with_partitions = Expander.new.call(topics)
45
45
 
46
46
  @routing_topics = @topics_with_partitions.to_h do |name, _|
47
- [name, ::Karafka::Routing::Router.find_or_initialize_by_name(name)]
47
+ [name, Karafka::Routing::Router.find_or_initialize_by_name(name)]
48
48
  end
49
49
 
50
50
  @total_partitions = @topics_with_partitions.map(&:last).sum(&:count)
@@ -120,7 +120,7 @@ module Karafka
120
120
  # Loads the Pro features of Karafka
121
121
  # @note Object space lookup is not the fastest but we do it once during boot, so it's ok
122
122
  def load_topic_features
123
- ::Karafka::Pro::Routing::Features::Base.load_all
123
+ Karafka::Pro::Routing::Features::Base.load_all
124
124
  end
125
125
  end
126
126
  end
@@ -8,7 +8,7 @@ module Karafka
8
8
  module Processing
9
9
  # Pro coordinator that provides extra orchestration methods useful for parallel processing
10
10
  # within the same partition
11
- class Coordinator < ::Karafka::Processing::Coordinator
11
+ class Coordinator < Karafka::Processing::Coordinator
12
12
  extend Forwardable
13
13
  include Helpers::ConfigImporter.new(
14
14
  errors_tracker_class: %i[internal processing errors_tracker_class]
@@ -20,6 +20,7 @@ module Karafka
20
20
 
21
21
  include Karafka::Core::Helpers::Time
22
22
 
23
+ # Initializes the filter as not yet applied
23
24
  def initialize
24
25
  @applied = false
25
26
  @cursor = nil
@@ -25,7 +25,7 @@ module Karafka
25
25
 
26
26
  # Time on message is in seconds with ms precision, so we need to convert the ttl that
27
27
  # is in ms to this format
28
- border = ::Time.now.utc - (@delay / 1_000.0)
28
+ border = Time.now.utc - (@delay / 1_000.0)
29
29
 
30
30
  messages.delete_if do |message|
31
31
  too_young = message.timestamp > border
@@ -26,7 +26,7 @@ module Karafka
26
26
 
27
27
  # Time on message is in seconds with ms precision, so we need to convert the ttl that
28
28
  # is in ms to this format
29
- border = ::Time.now.utc - (@ttl / 1_000.to_f)
29
+ border = Time.now.utc - (@ttl / 1_000.to_f)
30
30
 
31
31
  messages.delete_if do |message|
32
32
  too_old = message.timestamp < border
@@ -41,7 +41,7 @@ module Karafka
41
41
  # This can happen when we chain filters
42
42
  return unless @cursor
43
43
 
44
- insights = ::Karafka::Processing::InlineInsights::Tracker.find(
44
+ insights = Karafka::Processing::InlineInsights::Tracker.find(
45
45
  @topic,
46
46
  @partition
47
47
  )
@@ -17,7 +17,7 @@ module Karafka
17
17
  #
18
18
  # @note It needs to be working with a proper consumer that will handle the partition
19
19
  # management. This layer of the framework knows nothing about Kafka messages consumption.
20
- class ConsumeNonBlocking < ::Karafka::Processing::Jobs::Consume
20
+ class ConsumeNonBlocking < Karafka::Processing::Jobs::Consume
21
21
  self.action = :consume
22
22
 
23
23
  # Makes this job non-blocking from the start
@@ -11,7 +11,7 @@ module Karafka
11
11
  # We use this version for LRJ topics for cases where saturated resources would not allow
12
12
  # to run this job for extended period of time. Under such scenarios, if we would not use
13
13
  # a non-blocking one, we would reach max.poll.interval.ms.
14
- class EofedNonBlocking < ::Karafka::Processing::Jobs::Eofed
14
+ class EofedNonBlocking < Karafka::Processing::Jobs::Eofed
15
15
  self.action = :eofed
16
16
 
17
17
  # @param args [Array] any arguments accepted by `::Karafka::Processing::Jobs::Eofed`
@@ -9,7 +9,7 @@ module Karafka
9
9
  module Jobs
10
10
  # Job that represents a "ticking" work. Work that we run periodically for the Periodics
11
11
  # enabled topics.
12
- class Periodic < ::Karafka::Processing::Jobs::Base
12
+ class Periodic < Karafka::Processing::Jobs::Base
13
13
  self.action = :tick
14
14
 
15
15
  # @param executor [Karafka::Pro::Processing::Executor] pro executor that is suppose to
@@ -15,7 +15,7 @@ module Karafka
15
15
  # It can be useful when having long lasting jobs that would exceed `max.poll.interval`
16
16
  # in scenarios where there are more jobs than threads, without this being async we
17
17
  # would potentially stop polling
18
- class RevokedNonBlocking < ::Karafka::Processing::Jobs::Revoked
18
+ class RevokedNonBlocking < Karafka::Processing::Jobs::Revoked
19
19
  self.action = :revoked
20
20
 
21
21
  # Makes this job non-blocking from the start
@@ -7,7 +7,7 @@ module Karafka
7
7
  module Pro
8
8
  module Processing
9
9
  # Pro jobs builder that supports lrj
10
- class JobsBuilder < ::Karafka::Processing::JobsBuilder
10
+ class JobsBuilder < Karafka::Processing::JobsBuilder
11
11
  # @param executor [Karafka::Pro::Processing::Executor]
12
12
  def idle(executor)
13
13
  Karafka::Processing::Jobs::Idle.new(executor)
@@ -101,10 +101,8 @@ module Karafka
101
101
  end
102
102
 
103
103
  # Allows for explicit unlocking of locked queue of a group
104
- #
105
104
  # @param group_id [String] id of the group we want to unlock
106
105
  # @param lock_id [Object] unique id we want to use to identify our lock
107
- #
108
106
  def unlock_async(group_id, lock_id)
109
107
  @mutex.synchronize do
110
108
  if @locks[group_id].delete(lock_id)
@@ -28,6 +28,7 @@ module Karafka
28
28
  def_delegators :instance, :register, :clear, :find
29
29
  end
30
30
 
31
+ # Initializes the fetcher with empty caches
31
32
  def initialize
32
33
  @mutexes = {}
33
34
  @clients = {}
@@ -7,7 +7,7 @@ module Karafka
7
7
  module Pro
8
8
  module Processing
9
9
  # Pro partitioner that can distribute work based on the virtual partitioner settings
10
- class Partitioner < ::Karafka::Processing::Partitioner
10
+ class Partitioner < Karafka::Processing::Partitioner
11
11
  # @param topic [String] topic name
12
12
  # @param messages [Array<Karafka::Messages::Message>] karafka messages
13
13
  # @param coordinator [Karafka::Pro::Processing::Coordinator] processing coordinator that
@@ -10,7 +10,7 @@ module Karafka
10
10
  module Strategies
11
11
  # Base strategy for Pro
12
12
  module Base
13
- include ::Karafka::Processing::Strategies::Base
13
+ include Karafka::Processing::Strategies::Base
14
14
  end
15
15
  end
16
16
  end
@@ -14,7 +14,7 @@ module Karafka
14
14
  # Nothing. Just standard, automatic flow
15
15
  module Default
16
16
  include Base
17
- include ::Karafka::Processing::Strategies::Default
17
+ include Karafka::Processing::Strategies::Default
18
18
 
19
19
  # Apply strategy for a non-feature based flow
20
20
  FEATURES = %i[].freeze
@@ -263,7 +263,7 @@ module Karafka
263
263
  # the post-user code execution marking with transactional producer to result in a
264
264
  # boolean state of marking for further framework flow. This is a normalization to make it
265
265
  # behave the same way as it would behave with a non-transactional one
266
- rescue ::Rdkafka::RdkafkaError, Errors::AssignmentLostError
266
+ rescue Rdkafka::RdkafkaError, Errors::AssignmentLostError
267
267
  false
268
268
  ensure
269
269
  @_transaction_internal = false
@@ -81,7 +81,7 @@ module Karafka
81
81
  # @return [Array<Karafka::Messages::Message, Boolean>] message we may want to skip and
82
82
  # information if this message was from marked offset or figured out via mom flow
83
83
  def find_skippable_message
84
- skippable_message = messages.find do |msg|
84
+ skippable_message = messages.raw.find do |msg|
85
85
  coordinator.marked? && msg.offset == seek_offset
86
86
  end
87
87
 
@@ -156,7 +156,7 @@ module Karafka
156
156
  return if collapsed?
157
157
 
158
158
  coordinator.virtual_offset_manager.register(
159
- messages.map(&:offset)
159
+ messages.raw.map(&:offset)
160
160
  )
161
161
  end
162
162
  end
@@ -23,6 +23,7 @@ module Karafka
23
23
  filtering
24
24
  ].freeze
25
25
 
26
+ # Initializes the strategy selector and preloads all strategies
26
27
  def initialize
27
28
  # Preload the strategies
28
29
  # We load them once for performance reasons not to do too many lookups
@@ -11,8 +11,10 @@ module Karafka
11
11
  # Balanced distributor that groups messages by partition key
12
12
  # and processes larger groups first while maintaining message order within groups
13
13
  class Balanced < Base
14
- # @param messages [Array<Karafka::Messages::Message>] messages to distribute
15
- # @return [Hash<Integer, Array<Karafka::Messages::Message>>] hash with group ids as
14
+ # Distributes messages to virtual partitions ensuring balanced load across workers
15
+ # by grouping messages by partition key and assigning larger groups first
16
+ # @param messages [Array<Karafka::Messages::Message>]
17
+ # @return [Hash{Integer => Array<Karafka::Messages::Message>}] hash with group ids as
16
18
  # keys and message groups as values
17
19
  def call(messages)
18
20
  # Group messages by partition key
@@ -11,8 +11,10 @@ module Karafka
11
11
  # Consistent distributor that ensures messages with the same partition key
12
12
  # are always processed in the same virtual partition
13
13
  class Consistent < Base
14
- # @param messages [Array<Karafka::Messages::Message>] messages to distribute
15
- # @return [Hash<Integer, Array<Karafka::Messages::Message>>] hash with group ids as
14
+ # Distributes messages ensuring consistent routing where messages with the same
15
+ # partition key always go to the same virtual partition
16
+ # @param messages [Array<Karafka::Messages::Message>]
17
+ # @return [Hash{Integer => Array<Karafka::Messages::Message>}] hash with group ids as
16
18
  # keys and message groups as values
17
19
  def call(messages)
18
20
  messages
@@ -11,13 +11,14 @@ module Karafka
11
11
  # - we only run schedules that are of same or newer version
12
12
  # - we always mark as consumed in such a way, that the first message received after
13
13
  # assignment (if any) is a state
14
- class Consumer < ::Karafka::BaseConsumer
14
+ class Consumer < Karafka::BaseConsumer
15
15
  # @param args [Array] all arguments accepted by the consumer
16
16
  def initialize(*args)
17
17
  super
18
18
  @executor = Executor.new
19
19
  end
20
20
 
21
+ # Consumes messages and manages recurring tasks execution
21
22
  def consume
22
23
  # There is nothing we can do if we operate on a newer schedule. In such cases we should
23
24
  # just wait and re-raise error hoping someone will notice or that this will be
@@ -54,7 +55,7 @@ module Karafka
54
55
  # that collectively have a different outcome
55
56
  @executor.call
56
57
  else
57
- raise ::Karafka::Errors::UnsupportedCaseError, type
58
+ raise Karafka::Errors::UnsupportedCaseError, type
58
59
  end
59
60
  end
60
61
 
@@ -9,7 +9,7 @@ module Karafka
9
9
  # Recurring Tasks related contracts
10
10
  module Contracts
11
11
  # Makes sure, all the expected config is defined as it should be
12
- class Config < ::Karafka::Contracts::Base
12
+ class Config < Karafka::Contracts::Base
13
13
  configure do |config|
14
14
  config.error_messages = YAML.safe_load_file(
15
15
  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
@@ -17,7 +17,7 @@ module Karafka
17
17
  end
18
18
 
19
19
  nested(:recurring_tasks) do
20
- required(:consumer_class) { |val| val < ::Karafka::BaseConsumer }
20
+ required(:consumer_class) { |val| val < Karafka::BaseConsumer }
21
21
  required(:deserializer) { |val| !val.nil? }
22
22
  required(:logging) { |val| [true, false].include?(val) }
23
23
  # Do not allow to run more often than every 5 seconds
@@ -9,7 +9,7 @@ module Karafka
9
9
  # Recurring Tasks related contracts
10
10
  module Contracts
11
11
  # Ensures that task details are as expected
12
- class Task < ::Karafka::Contracts::Base
12
+ class Task < Karafka::Contracts::Base
13
13
  configure do |config|
14
14
  config.error_messages = YAML.safe_load_file(
15
15
  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
@@ -16,7 +16,7 @@ module Karafka
16
16
  # @param message [::Karafka::Messages::Message]
17
17
  # @return [Hash] deserialized data
18
18
  def call(message)
19
- ::JSON.parse(
19
+ JSON.parse(
20
20
  Zlib::Inflate.inflate(message.raw_payload),
21
21
  symbolize_names: true
22
22
  )
@@ -51,7 +51,7 @@ module Karafka
51
51
  # @note We do not fetch it via the ConfigImporter not to cache it so we can re-use it
52
52
  # if needed
53
53
  def producer
54
- ::Karafka::App.config.recurring_tasks.producer
54
+ Karafka::App.config.recurring_tasks.producer
55
55
  end
56
56
 
57
57
  # @return [Serializer]
@@ -17,6 +17,7 @@ module Karafka
17
17
  trigger
18
18
  ].freeze
19
19
 
20
+ # Initializes the executor in replaying mode
20
21
  def initialize
21
22
  @replaying = true
22
23
  @incompatible = false
@@ -131,7 +132,7 @@ module Karafka
131
132
 
132
133
  # @return [Karafka::Pro::RecurringTasks::Schedule] current in-memory schedule
133
134
  def schedule
134
- ::Karafka::Pro::RecurringTasks.schedule
135
+ Karafka::Pro::RecurringTasks.schedule
135
136
  end
136
137
 
137
138
  # Dispatches the current schedule state to Kafka
@@ -13,7 +13,7 @@ module Karafka
13
13
  # @return [String]
14
14
  attr_reader :version
15
15
 
16
- # @return [Hash<String, Task>]
16
+ # @return [Hash{String => Task}]
17
17
  attr_reader :tasks
18
18
 
19
19
  # @param version [String] schedule version. In case of usage of versioning it is used to
@@ -43,7 +43,10 @@ module Karafka
43
43
  end
44
44
 
45
45
  # Allows us to have a nice DSL for defining schedules
46
- # @param args [Array] attributes accepted by the task initializer
46
+ # @param args [Hash] attributes accepted by the task initializer
47
+ # @option args [String] :id unique task identifier
48
+ # @option args [String] :cron cron expression for task scheduling
49
+ # @option args [Proc] :previous_time optional lambda returning previous execution time
47
50
  def schedule(**args, &)
48
51
  self << Task.new(**args, &)
49
52
  end
@@ -11,7 +11,8 @@ module Karafka
11
11
  # Current recurring tasks related schema structure
12
12
  SCHEMA_VERSION = '1.0'
13
13
 
14
- # @param schedule [Karafka::Pro::RecurringTasks::Schedule] schedule to serialize
14
+ # Serializes and compresses the schedule with all its tasks and their execution state
15
+ # @param schedule [Karafka::Pro::RecurringTasks::Schedule]
15
16
  # @return [String] serialized and compressed current schedule data with its tasks and their
16
17
  # current state.
17
18
  def schedule(schedule)
@@ -46,7 +47,7 @@ module Karafka
46
47
  def command(command_name, task_id)
47
48
  data = {
48
49
  schema_version: SCHEMA_VERSION,
49
- schedule_version: ::Karafka::Pro::RecurringTasks.schedule.version,
50
+ schedule_version: Karafka::Pro::RecurringTasks.schedule.version,
50
51
  dispatched_at: Time.now.to_f,
51
52
  type: 'command',
52
53
  command: {
@@ -69,7 +70,7 @@ module Karafka
69
70
 
70
71
  data = {
71
72
  schema_version: SCHEMA_VERSION,
72
- schedule_version: ::Karafka::Pro::RecurringTasks.schedule.version,
73
+ schedule_version: Karafka::Pro::RecurringTasks.schedule.version,
73
74
  dispatched_at: Time.now.to_f,
74
75
  type: 'log',
75
76
  task: {
@@ -92,9 +93,9 @@ module Karafka
92
93
  hash.to_json
93
94
  end
94
95
 
95
- # Compresses the provided data
96
+ # Compresses the provided data using Zlib deflate algorithm
96
97
  #
97
- # @param data [String] data to compress
98
+ # @param data [String]
98
99
  # @return [String] compressed data
99
100
  def compress(data)
100
101
  Zlib::Deflate.deflate(data)