karafka 2.5.1 → 2.5.2

data/lib/karafka/pro/active_job/dispatcher.rb

@@ -11,6 +11,10 @@ module Karafka
       # and that allows to inject additional options into the producer, effectively allowing for a
       # much better and more granular control over the dispatch and consumption process.
       class Dispatcher < ::Karafka::ActiveJob::Dispatcher
+        include Helpers::ConfigImporter.new(
+          deserializer: %i[internal active_job deserializer]
+        )
+
         # Defaults for dispatching
         # They can be updated by using `#karafka_options` on the job
         DEFAULTS = {
@@ -43,7 +47,7 @@ module Karafka
             fetch_option(job, :dispatch_method, DEFAULTS),
             dispatch_details(job).merge!(
               topic: job.queue_name,
-              payload: ::ActiveSupport::JSON.encode(serialize_job(job))
+              payload: serialize_job(job)
             )
           )
         end
@@ -64,7 +68,7 @@ module Karafka
 
             dispatches[d_method][producer] << dispatch_details(job).merge!(
               topic: job.queue_name,
-              payload: ::ActiveSupport::JSON.encode(serialize_job(job))
+              payload: serialize_job(job)
             )
           end
 
@@ -90,7 +94,7 @@ module Karafka
 
           target_message = dispatch_details(job).merge!(
             topic: job.queue_name,
-            payload: ::ActiveSupport::JSON.encode(serialize_job(job))
+            payload: serialize_job(job)
           )
 
           proxy_message = Pro::ScheduledMessages.schedule(

data/lib/karafka/pro/active_job/job_options_contract.rb

@@ -10,10 +10,8 @@ module Karafka
       # class that works with Pro features.
       class JobOptionsContract < Contracts::Base
         configure do |config|
-          config.error_messages = YAML.safe_load(
-            File.read(
-              File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
-            )
+          config.error_messages = YAML.safe_load_file(
+            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
           ).fetch('en').fetch('validations').fetch('job_options')
         end
 

data/lib/karafka/pro/cleaner/messages/messages.rb

@@ -17,7 +17,6 @@ module Karafka
         module Messages
           # @param clean [Boolean] do we want to clean each message after we're done working with
           #   it.
-          # @param block [Proc] block we want to execute per each message
           #
           # @note Cleaning messages after we're done with each of them and did not fail does not
           #   affect any other functionalities. The only thing that is crucial is to make sure,
@@ -27,14 +26,14 @@ module Karafka
           # @note This method calls super() to ensure compatibility with other libraries that
           #   may have prepended modules to modify #each behavior. This preserves the method
           #   chain and allows instrumentation libraries to function correctly.
-          def each(clean: false, &block)
+          def each(clean: false, &)
             if clean
               super() do |message|
                 yield(message)
                 message.clean!
               end
             else
-              super(&block)
+              super(&)
             end
           end
         end

data/lib/karafka/pro/cli/contracts/server.rb

@@ -12,10 +12,8 @@ module Karafka
         # It differs slightly from the OSS one because it is aware of the routing patterns
         class Server < ::Karafka::Cli::Contracts::Server
           configure do |config|
-            config.error_messages = YAML.safe_load(
-              File.read(
-                File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
-              )
+            config.error_messages = YAML.safe_load_file(
+              File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
             ).fetch('en').fetch('validations').fetch('cli').fetch('server')
           end
 

data/lib/karafka/pro/cli/parallel_segments/base.rb

@@ -67,8 +67,7 @@ module Karafka
             consumer_groups = [segment_origin, segments.map(&:name)].flatten
 
             consumer_groups_with_topics = consumer_groups
-                                          .map { |name| [name, topics_names] }
-                                          .to_h
+                                          .to_h { |name| [name, topics_names] }
 
             lags_with_offsets = Karafka::Admin.read_lags_with_offsets(
               consumer_groups_with_topics

data/lib/karafka/pro/cli/parallel_segments/collapse.rb

@@ -123,8 +123,8 @@ module Karafka
                 inconclusive = true
 
                 puts(
-                  "  Inconclusive offsets for #{red(topic_name)}##{red(partition_id)}:" \
-                  " #{parallel_offsets.to_a.join(', ')}"
+                  "  Inconclusive offsets for #{red(topic_name)}##{red(partition_id)}: " \
+                  "#{parallel_offsets.to_a.join(', ')}"
                 )
               end
             end

data/lib/karafka/pro/cli/parallel_segments/distribute.rb

@@ -92,8 +92,8 @@ module Karafka
 
                   raise(
                     ::Karafka::Errors::CommandValidationError,
-                    "Parallel segment #{red(cg_name)} already has offset #{red(offset)}" \
-                    " set for #{red("#{topic_name}##{partition_id}")}"
+                    "Parallel segment #{red(cg_name)} already has offset #{red(offset)} " \
+                    "set for #{red("#{topic_name}##{partition_id}")}"
                   )
                 end
               end

data/lib/karafka/pro/connection/manager.rb

@@ -21,7 +21,7 @@ module Karafka
 
         # Creates new manager instance
         def initialize
-          super()
+          super
           @mutex = Mutex.new
           @changes = Hash.new do |h, k|
             h[k] = {
@@ -69,7 +69,7 @@ module Karafka
           times = []
           # stateage is in microseconds
           # We monitor broker changes to make sure we do not introduce extra friction
-          times << statistics['brokers'].values.map { |stats| stats['stateage'] }.min / 1_000
+          times << (statistics['brokers'].values.map { |stats| stats['stateage'] }.min / 1_000)
           times << statistics['cgrp']['rebalance_age']
           times << statistics['cgrp']['stateage']
 

data/lib/karafka/pro/encryption/contracts/config.rb

@@ -11,10 +11,8 @@ module Karafka
         # Makes sure, all the expected config is defined as it should be
         class Config < ::Karafka::Contracts::Base
           configure do |config|
-            config.error_messages = YAML.safe_load(
-              File.read(
-                File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
-              )
+            config.error_messages = YAML.safe_load_file(
+              File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
             ).fetch('en').fetch('validations').fetch('setup').fetch('config')
           end
 
@@ -26,8 +24,8 @@ module Karafka
 
             required(:private_keys) do |val|
               val.is_a?(Hash) &&
-                val.keys.all? { |key| key.is_a?(String) } &&
-                val.values.all? { |key| key.is_a?(String) }
+                val.keys.all?(String) &&
+                val.values.all?(String)
             end
           end
 

data/lib/karafka/pro/encryption/messages/parser.rb

@@ -25,7 +25,7 @@ module Karafka
             encryption = headers['encryption']
             fingerprint = headers['encryption_fingerprint']
 
-            return super(message) unless active && encryption
+            return super unless active && encryption
 
             # Decrypt raw payload so it can be handled by the default parser logic
             decrypted_payload = cipher.decrypt(
@@ -35,11 +35,11 @@ module Karafka
 
             message.raw_payload = decrypted_payload
 
-            return super(message) unless fingerprint && fingerprinter
+            return super unless fingerprint && fingerprinter
 
             message_fingerprint = fingerprinter.hexdigest(decrypted_payload)
 
-            return super(message) if message_fingerprint == fingerprint
+            return super if message_fingerprint == fingerprint
 
             raise(Errors::FingerprintVerificationError, message.to_s)
           end

data/lib/karafka/pro/instrumentation/performance_tracker.rb

@@ -50,7 +50,7 @@ module Karafka
           partition = messages.metadata.partition
 
           samples = @processing_times[topic][partition]
-          samples << event[:time] / messages.size
+          samples << (event[:time] / messages.size)
 
           return unless samples.size > SAMPLES_COUNT
 
@@ -66,8 +66,8 @@ module Karafka
         def percentile(percentile, values)
           values_sorted = values.sort
 
-          floor = (percentile * (values_sorted.length - 1) + 1).floor - 1
-          mod = (percentile * (values_sorted.length - 1) + 1).modulo(1)
+          floor = ((percentile * (values_sorted.length - 1)) + 1).floor - 1
+          mod = ((percentile * (values_sorted.length - 1)) + 1).modulo(1)
 
           values_sorted[floor] + (mod * (values_sorted[floor + 1] - values_sorted[floor]))
         end

data/lib/karafka/pro/iterator/expander.rb

@@ -62,7 +62,7 @@ module Karafka
 
           # If we've got just array with topics, we need to convert that into a representation
           # that we can expand with offsets
-          topics = topics.map { |name| [name, false] }.to_h if topics.is_a?(Array)
+          topics = topics.to_h { |name| [name, false] } if topics.is_a?(Array)
           # We remap by creating new hash, just in case the hash came as the argument for this
           # expanded. We do not want to modify user provided hash
           topics.transform_keys(&:to_s)

data/lib/karafka/pro/iterator/tpl_builder.rb

@@ -56,7 +56,7 @@ module Karafka
             next unless partitions.is_a?(Array) || partitions.is_a?(Range)
 
             # When no offsets defined, we just start from zero
-            @mapped_topics[name] = partitions.map { |partition| [partition, 0] }.to_h
+            @mapped_topics[name] = partitions.to_h { |partition| [partition, 0] }
           end
         end
 

data/lib/karafka/pro/iterator.rb

@@ -43,9 +43,9 @@ module Karafka
       )
         @topics_with_partitions = Expander.new.call(topics)
 
-        @routing_topics = @topics_with_partitions.map do |name, _|
+        @routing_topics = @topics_with_partitions.to_h do |name, _|
           [name, ::Karafka::Routing::Router.find_or_initialize_by_name(name)]
-        end.to_h
+        end
 
         @total_partitions = @topics_with_partitions.map(&:last).sum(&:count)
 

data/lib/karafka/pro/processing/coordinators/errors_tracker.rb

@@ -79,9 +79,8 @@ module Karafka
           end
 
           # Iterates over errors
-          # @param block [Proc] code we want to run on each error
-          def each(&block)
-            @errors.each(&block)
+          def each(&)
+            @errors.each(&)
           end
 
           # @return [Array<StandardError>] array with all the errors that occurred

data/lib/karafka/pro/processing/coordinators/filters_applier.rb

@@ -68,7 +68,7 @@ module Karafka
           # @return [Integer] minimum timeout we need to pause. This is the minimum for all the
           #   filters to satisfy all of them.
           def timeout
-            applied.map(&:timeout).compact.min || 0
+            applied.filter_map(&:timeout).min || 0
           end
 
           # The first message we do need to get next time we poll. We use the minimum not to jump
@@ -78,7 +78,7 @@ module Karafka
           def cursor
             return nil unless active?
 
-            applied.map(&:cursor).compact.min_by(&:offset)
+            applied.filter_map(&:cursor).min_by(&:offset)
           end
 
           # @return [Boolean] did any of the filters requested offset storage during filter
@@ -106,7 +106,7 @@ module Karafka
           def marking_cursor
             return nil unless active?
 
-            applied.map(&:marking_cursor).compact.min_by(&:offset)
+            applied.filter_map(&:marking_cursor).min_by(&:offset)
           end
 
           private

data/lib/karafka/pro/processing/filters/delayer.rb

@@ -25,7 +25,7 @@ module Karafka
 
             # Time on message is in seconds with ms precision, so we need to convert the ttl that
             # is in ms to this format
-            border = ::Time.now.utc - @delay / 1_000.0
+            border = ::Time.now.utc - (@delay / 1_000.0)
 
             messages.delete_if do |message|
               too_young = message.timestamp > border

data/lib/karafka/pro/processing/filters/expirer.rb

@@ -26,7 +26,7 @@ module Karafka
 
             # Time on message is in seconds with ms precision, so we need to convert the ttl that
             # is in ms to this format
-            border = ::Time.now.utc - @ttl / 1_000.to_f
+            border = ::Time.now.utc - (@ttl / 1_000.to_f)
 
             messages.delete_if do |message|
               too_old = message.timestamp < border

data/lib/karafka/pro/processing/filters/throttler.rb

@@ -67,7 +67,7 @@ module Karafka
           #   so we are no longer throttled and so we can process at least one message
           def timeout
             timeout = @interval - (monotonic_now - @time)
-            timeout <= 0 ? 0 : timeout
+            [timeout, 0].max
           end
         end
       end

data/lib/karafka/pro/processing/schedulers/default.rb

@@ -29,10 +29,8 @@ module Karafka
           def on_schedule_consumption(jobs_array)
             perf_tracker = Instrumentation::PerformanceTracker.instance
 
-            ordered = []
-
-            jobs_array.each do |job|
-              ordered << [
+            ordered = jobs_array.map do |job|
+              [
                 job,
                 processing_cost(perf_tracker, job)
               ]

data/lib/karafka/pro/processing/strategies/lrj/default.rb

@@ -78,10 +78,8 @@ module Karafka
             # Allows for LRJ to synchronize its work. It may be needed because LRJ can run
             # lifecycle events like revocation while the LRJ work is running and there may be a
             # need for a critical section.
-            #
-            # @param block [Proc] block we want to run in a mutex to prevent race-conditions
-            def synchronize(&block)
-              coordinator.shared_mutex.synchronize(&block)
+            def synchronize(&)
+              coordinator.shared_mutex.synchronize(&)
             end
           end
         end

data/lib/karafka/pro/processing/strategies/vp/default.rb

@@ -134,10 +134,8 @@ module Karafka
             # the end users. With LRJ it is needed and provided in the `LRJ::Default` strategy,
             # because lifecycle events on revocation can run in parallel to the LRJ job as it is
             # non-blocking.
-            #
-            # @param block [Proc] block we want to run in a mutex to prevent race-conditions
-            def synchronize(&block)
-              coordinator.shared_mutex.synchronize(&block)
+            def synchronize(&)
+              coordinator.shared_mutex.synchronize(&)
             end
 
             private

data/lib/karafka/pro/processing/subscription_groups_coordinator.rb

@@ -16,12 +16,11 @@ module Karafka
         #   want to pause
         # @param lock_id [Object] key we want to use if we want to set multiple locks on the same
         #   subscription group
-        # @param kwargs [Object] Any keyword arguments accepted by the jobs queue lock.
-        def pause(subscription_group, lock_id = nil, **kwargs)
+        def pause(subscription_group, lock_id = nil, **)
           jobs_queue.lock_async(
             subscription_group.id,
             lock_id,
-            **kwargs
+            **
           )
         end
 

data/lib/karafka/pro/recurring_tasks/contracts/config.rb

@@ -11,10 +11,8 @@ module Karafka
         # Makes sure, all the expected config is defined as it should be
         class Config < ::Karafka::Contracts::Base
           configure do |config|
-            config.error_messages = YAML.safe_load(
-              File.read(
-                File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
-              )
+            config.error_messages = YAML.safe_load_file(
+              File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
             ).fetch('en').fetch('validations').fetch('setup').fetch('config')
           end
 

data/lib/karafka/pro/recurring_tasks/contracts/task.rb

@@ -11,10 +11,8 @@ module Karafka
         # Ensures that task details are as expected
         class Task < ::Karafka::Contracts::Base
           configure do |config|
-            config.error_messages = YAML.safe_load(
-              File.read(
-                File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
-              )
+            config.error_messages = YAML.safe_load_file(
+              File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
             ).fetch('en').fetch('validations').fetch('recurring_tasks')
           end
 

data/lib/karafka/pro/recurring_tasks/dispatcher.rb

@@ -8,6 +8,10 @@ module Karafka
     module RecurringTasks
       # Dispatches appropriate recurring tasks related messages to expected topics
       class Dispatcher
+        extend Helpers::ConfigImporter.new(
+          topics: %i[recurring_tasks topics]
+        )
+
         class << self
           # Snapshots to Kafka current schedule state
           def schedule
@@ -44,15 +48,12 @@ module Karafka
           private
 
           # @return [::WaterDrop::Producer] web ui producer
+          # @note We do not fetch it via the ConfigImporter not to cache it so we can re-use it
+          #   if needed
           def producer
             ::Karafka::App.config.recurring_tasks.producer
           end
 
-          # @return [String] consumers commands topic
-          def topics
-            ::Karafka::App.config.recurring_tasks.topics
-          end
-
           # @return [Serializer]
           def serializer
             Serializer.new

data/lib/karafka/pro/recurring_tasks/schedule.rb

@@ -32,9 +32,8 @@ module Karafka
         end
 
         # Iterates over tasks yielding them one after another
-        # @param block [Proc] block that will be executed with each task
-        def each(&block)
-          @tasks.each_value(&block)
+        def each(&)
+          @tasks.each_value(&)
         end
 
         # @param id [String] id of a particular recurring task
@@ -45,9 +44,8 @@ module Karafka
 
         # Allows us to have a nice DSL for defining schedules
         # @param args [Array] attributes accepted by the task initializer
-        # @param block [Proc] block to execute
-        def schedule(**args, &block)
-          self << Task.new(**args, &block)
+        def schedule(**args, &)
+          self << Task.new(**args, &)
         end
       end
     end

data/lib/karafka/pro/recurring_tasks.rb

@@ -7,16 +7,19 @@ module Karafka
   module Pro
     # Recurring tasks functionality
     module RecurringTasks
+      extend Helpers::ConfigImporter.new(
+        recurring_tasks_logging: %i[recurring_tasks logging]
+      )
+
       class << self
         # @return [Schedule, nil] current defined schedule or nil if not defined
         def schedule
-          @schedule || define('0.0.0') {}
+          @schedule || define('0.0.0') { nil }
         end
 
         # Simplified API for schedules definitions and validates the tasks data
         #
         # @param version [String]
-        # @param block [Proc]
         #
         # @example
         #   Karafka::Pro::RecurringTasks.define('1.0.1') do
@@ -24,9 +27,9 @@ module Karafka
         #       MailingJob.perform_async
         #     end
         #   end
-        def define(version = '1.0.0', &block)
+        def define(version = '1.0.0', &)
           @schedule = Schedule.new(version: version)
-          @schedule.instance_exec(&block)
+          @schedule.instance_exec(&)
 
           @schedule.each do |task|
             Contracts::Task.new.validate!(
@@ -75,7 +78,7 @@ module Karafka
           RecurringTasks.schedule
 
           # User can disable logging of executions, in which case we don't track them
-          return unless Karafka::App.config.recurring_tasks.logging
+          return unless recurring_tasks_logging
 
           Karafka.monitor.subscribe(Listener.new)
         end

data/lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb

@@ -13,10 +13,8 @@ module Karafka
             # Contract to validate configuration of the adaptive iterator feature
             class Topic < Karafka::Contracts::Base
               configure do |config|
-                config.error_messages = YAML.safe_load(
-                  File.read(
-                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
-                  )
+                config.error_messages = YAML.safe_load_file(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
                 ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
               end
 

data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb

@@ -13,10 +13,8 @@ module Karafka
             # Extended rules for dead letter queue settings
             class Topic < Karafka::Contracts::Base
               configure do |config|
-                config.error_messages = YAML.safe_load(
-                  File.read(
-                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
-                  )
+                config.error_messages = YAML.safe_load_file(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
                 ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
               end
 

data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb

@@ -13,10 +13,8 @@ module Karafka
             # Contract to validate configuration of the expiring feature
             class Topic < Karafka::Contracts::Base
               configure do |config|
-                config.error_messages = YAML.safe_load(
-                  File.read(
-                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
-                  )
+                config.error_messages = YAML.safe_load_file(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
                 ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
               end
 

data/lib/karafka/pro/routing/features/delaying/topic.rb

@@ -35,10 +35,8 @@ module Karafka
             end
 
             # Just an alias for nice API
-            #
-            # @param args [Array] Anything `#delaying` accepts
-            def delay_by(*args)
-              delaying(*args)
+            def delay_by(*)
+              delaying(*)
             end
 
             # @return [Boolean] is a given job delaying

data/lib/karafka/pro/routing/features/direct_assignments/contracts/consumer_group.rb

@@ -12,19 +12,15 @@ module Karafka
             # Contract to validate configuration of the direct assignments feature
             class ConsumerGroup < Karafka::Contracts::Base
               configure do |config|
-                config.error_messages = YAML.safe_load(
-                  File.read(
-                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
-                  )
+                config.error_messages = YAML.safe_load_file(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
                 ).fetch('en').fetch('validations').fetch('routing').fetch('consumer_group')
 
                 virtual do |data, errors|
                   next unless errors.empty?
 
-                  active = []
-
-                  data[:topics].each do |topic|
-                    active << topic[:direct_assignments][:active]
+                  active = data[:topics].map do |topic|
+                    topic[:direct_assignments][:active]
                   end
 
                   # If none active we use standard subscriptions

data/lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb

@@ -13,10 +13,8 @@ module Karafka
             # Contract to validate configuration of the direct assignments topic feature
             class Topic < Karafka::Contracts::Base
               configure do |config|
-                config.error_messages = YAML.safe_load(
-                  File.read(
-                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
-                  )
+                config.error_messages = YAML.safe_load_file(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
                 ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
               end
 
@@ -26,8 +24,8 @@ module Karafka
                 required(:partitions) do |val|
                   next true if val == true
                   next false unless val.is_a?(Hash)
-                  next false unless val.keys.all? { |part| part.is_a?(Integer) }
-                  next false unless val.values.all? { |flag| flag == true }
+                  next false unless val.keys.all?(Integer)
+                  next false unless val.values.all?(true)
 
                   true
                 end
@@ -67,7 +65,7 @@ module Karafka
                 direct_partitions = direct_assignments[:partitions].keys
                 swarm_partitions = nodes.values.flatten
 
-                next unless swarm_partitions.all? { |partition| partition.is_a?(Integer) }
+                next unless swarm_partitions.all?(Integer)
                 next if direct_partitions.sort == swarm_partitions.sort
 
                 # If we assigned more partitions than we distributed in swarm