karafka 2.5.2 → 2.5.3

data/lib/karafka/pro/scheduled_messages/contracts/config.rb

@@ -9,7 +9,7 @@ module Karafka
       # Recurring Tasks related contracts
       module Contracts
         # Makes sure, all the expected config is defined as it should be
-        class Config < ::Karafka::Contracts::Base
+        class Config < Karafka::Contracts::Base
           configure do |config|
             config.error_messages = YAML.safe_load_file(
               File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
@@ -17,7 +17,7 @@ module Karafka
           end
 
           nested(:scheduled_messages) do
-            required(:consumer_class) { |val| val < ::Karafka::BaseConsumer }
+            required(:consumer_class) { |val| val < Karafka::BaseConsumer }
 
             # Do not allow to run more often than every second
             required(:interval) { |val| val.is_a?(Integer) && val >= 1_000 }

data/lib/karafka/pro/scheduled_messages/contracts/message.rb

@@ -11,7 +11,7 @@ module Karafka
         #
         # Our envelope always needs to comply with this format, otherwise we won't have enough
         # details to be able to dispatch the message
-        class Message < ::Karafka::Contracts::Base
+        class Message < Karafka::Contracts::Base
           configure do |config|
             config.error_messages = YAML.safe_load_file(
               File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')

data/lib/karafka/pro/scheduled_messages/daily_buffer.rb

@@ -14,6 +14,7 @@ module Karafka
       # up to 1.5 second, thus it is acceptable. Please ping me if you encounter performance
       # issues with this naive implementation so it can be improved.
       class DailyBuffer
+        # Initializes the daily buffer with empty accumulator
         def initialize
           @accu = {}
         end
@@ -69,8 +70,8 @@ module Karafka
           selected.each { |_, message| yield(message) }
         end
 
-        # Removes given key from the accumulator
-        # @param key [String] key to remove
+        # Removes the schedule entry identified by the given key from the daily buffer
+        # @param key [String]
         def delete(key)
           @accu.delete(key)
         end

data/lib/karafka/pro/scheduled_messages/day.rb

@@ -18,6 +18,7 @@ module Karafka
         # @return [Integer] utc timestamp when this day starts. Equal to 00:00:00
         attr_reader :starts_at
 
+        # Initializes a day representation for the current UTC day
         def initialize
           @created_at = Time.now.to_i
 

data/lib/karafka/pro/scheduled_messages/deserializers/headers.rb

@@ -16,7 +16,7 @@ module Karafka
 
           private_constant :WORKABLE_TYPES
 
-          # @param metadata [Karafka::aMessages::Metadata]
+          # @param metadata [Karafka::Messages::Metadata]
           # @return [Hash] headers
           def call(metadata)
             raw_headers = metadata.raw_headers

data/lib/karafka/pro/scheduled_messages/deserializers/payload.rb

@@ -15,7 +15,7 @@ module Karafka
           # @param message [::Karafka::Messages::Message]
           # @return [Hash] deserialized data
           def call(message)
-            ::JSON.parse(
+            JSON.parse(
               Zlib::Inflate.inflate(message.raw_payload),
               symbolize_names: true
             )

data/lib/karafka/pro/scheduled_messages/max_epoch.rb

@@ -23,6 +23,7 @@ module Karafka
         # @return [Integer] max epoch recorded
         attr_reader :to_i
 
+        # Initializes max epoch tracker with -1 as starting value
         def initialize
           @max = -1
           @to_i = @max

data/lib/karafka/pro/scheduled_messages/proxy.rb

@@ -21,7 +21,7 @@ module Karafka
       module Proxy
         # General WaterDrop message contract. Before we envelop a message, we need to be certain
         # it is correct, hence we use this contract.
-        MSG_CONTRACT = ::WaterDrop::Contracts::Message.new(
+        MSG_CONTRACT = WaterDrop::Contracts::Message.new(
           # Payload size is a subject to the target producer dispatch validation, so we set it
           # to 100MB basically to ignore it here.
           max_payload_size: 104_857_600

data/lib/karafka/pro/scheduled_messages/serializer.rb

@@ -9,7 +9,7 @@ module Karafka
       # Serializers used to build payloads (if applicable) for dispatch
       # @note We only deal with states payload. Other payloads are not ours but end users.
       class Serializer
-        include ::Karafka::Core::Helpers::Time
+        include Karafka::Core::Helpers::Time
 
         # @param tracker [Tracker] tracker based on which we build the state
         # @return [String] compressed payload with the state details
@@ -32,9 +32,9 @@ module Karafka
           hash.to_json
         end
 
-        # Compresses the provided data
+        # Compresses the provided data using Zlib deflate algorithm
         #
-        # @param data [String] data to compress
+        # @param data [String]
         # @return [String] compressed data
         def compress(data)
           Zlib::Deflate.deflate(data)

data/lib/karafka/pro/scheduled_messages/setup/config.rb

@@ -10,7 +10,7 @@ module Karafka
       module Setup
         # Config for recurring tasks
         class Config
-          extend ::Karafka::Core::Configurable
+          extend Karafka::Core::Configurable
 
           setting(:consumer_class, default: Consumer)
           setting(:group_id, default: 'karafka_scheduled_messages')
@@ -27,7 +27,7 @@ module Karafka
           # Producer to use. By default uses default Karafka producer.
           setting(
             :producer,
-            constructor: -> { ::Karafka.producer },
+            constructor: -> { Karafka.producer },
             lazy: true
           )
 

data/lib/karafka/pro/scheduled_messages/state.rb

@@ -27,6 +27,7 @@ module Karafka
 
         private_constant :STATES
 
+        # Initializes the state as fresh
         def initialize
           @state = 'fresh'
         end

data/lib/karafka/pro/scheduled_messages/tracker.rb

@@ -18,6 +18,7 @@ module Karafka
         # @return [Integer] time epoch when this tracker was started
         attr_reader :started_at
 
+        # Initializes the tracker with empty statistics
         def initialize
           @daily = Hash.new { |h, k| h[k] = 0 }
           @started_at = Time.now.to_i

data/lib/karafka/process.rb

@@ -5,7 +5,7 @@ module Karafka
   # @note There might be only one process - this class is a singleton
   class Process
     # Allow for process tagging for instrumentation
-    extend ::Karafka::Core::Taggable
+    extend Karafka::Core::Taggable
 
     # Signal types that we handle
     HANDLED_SIGNALS = %i[
@@ -79,12 +79,12 @@ module Karafka
     private
 
     # Traps a single signal and performs callbacks (if any) or just ignores this signal
-    # @param [Symbol] signal type that we want to catch
+    # @param signal [Symbol] signal type that we want to catch
     # @note Since we do a lot of threading and queuing, we don't want to handle signals from the
     # trap context s some things may not work there as expected, that is why we spawn a separate
     # thread to handle the signals process
     def trap_signal(signal)
-      previous_handler = ::Signal.trap(signal) do
+      previous_handler = Signal.trap(signal) do
         Thread.new do
           notice_signal(signal)
 
@@ -96,7 +96,7 @@ module Karafka
     end
 
     # Informs monitoring about trapped signal
-    # @param [Symbol] signal type that we received
+    # @param signal [Symbol] signal type that we received
     def notice_signal(signal)
       Karafka.monitor.instrument('process.notice_signal', caller: self, signal: signal)
     end

data/lib/karafka/processing/executor.rb

@@ -184,7 +184,7 @@ module Karafka
           # We assign producer only when not available already. It may already be available if
           # user redefined the `#producer` method for example. This can be useful for example when
           # having a multi-cluster setup and using a totally custom producer
-          consumer.producer ||= ::Karafka::App.producer
+          consumer.producer ||= Karafka::App.producer
           # Since we have some message-less flows (idle, etc), we initialize consumer with empty
           # messages set. In production we have persistent consumers, so this is not a performance
           # overhead as this will happen only once per consumer lifetime

data/lib/karafka/processing/inline_insights/tracker.rb

@@ -51,6 +51,7 @@ module Karafka
           def_delegators :instance, :find, :add, :exists?, :clear
         end
 
+        # Initializes the tracker with empty accumulator
         def initialize
           @accu = {}
           @mutex = Mutex.new

data/lib/karafka/processing/jobs_queue.rb

@@ -99,7 +99,7 @@ module Karafka
       # Marks a given job from a given group as completed. When there are no more jobs from a given
       # group to be executed, we won't wait.
       #
-      # @param [Jobs::Base] job that was completed
+      # @param job [Jobs::Base] job that was completed
       def complete(job)
         @mutex.synchronize do
           # We finish one job and if there is another, we pick it up

data/lib/karafka/processing/result.rb

@@ -8,6 +8,7 @@ module Karafka
     class Result
       attr_reader :cause
 
+      # Initializes the result as successful with no cause
       def initialize
         @success = true
         @cause = false

data/lib/karafka/processing/strategy_selector.rb

@@ -13,6 +13,7 @@ module Karafka
         dead_letter_queue
       ].freeze
 
+      # Initializes the strategy selector and preloads all strategies
       def initialize
         # We load them once for performance reasons not to do too many lookups
         @strategies = find_all

data/lib/karafka/routing/activity_manager.rb

@@ -12,6 +12,7 @@ module Karafka
         topics
       ].freeze
 
+      # Initializes the activity manager with empty inclusion and exclusion lists
       def initialize
         @included = Hash.new { |h, k| h[k] = [] }
         @excluded = Hash.new { |h, k| h[k] = [] }

data/lib/karafka/routing/builder.rb

@@ -23,6 +23,7 @@ module Karafka
 
       private_constant :EMPTY_DEFAULTS
 
+      # Initializes the routing builder with empty routes
       def initialize
         @mutex = Mutex.new
         @draws = []
@@ -138,8 +139,9 @@ module Karafka
       # subscription group customization
       # @param subscription_group_name [String, Symbol] subscription group id. When not provided,
       #   a random uuid will be used
-      # @param args [Array] any extra arguments accepted by the subscription group builder
+      # @param args [Hash] any extra arguments accepted by the subscription group builder
       # @param block [Proc] further topics definitions
+      # @option args [String] :kafka optional kafka scope settings
       def subscription_group(
         subscription_group_name = SubscriptionGroup.id,
         **args,

data/lib/karafka/routing/contracts/consumer_group.rb

@@ -44,14 +44,15 @@ module Karafka
 
         virtual do |data, errors|
           next unless errors.empty?
-          next unless ::Karafka::App.config.strict_topics_namespacing
+          next unless Karafka::App.config.strict_topics_namespacing
 
           names = data.fetch(:topics).map { |topic| topic[:name] }
           names_hash = names.each_with_object({}) { |n, h| h[n] = true }
           error_occured = false
+          namespace_chars = ['.', '_'].freeze
           names.each do |n|
             # Skip topic names that are not namespaced
-            next unless n.chars.find { |c| ['.', '_'].include?(c) }
+            next unless n.chars.find { |c| namespace_chars.include?(c) }
 
             if n.chars.include?('.')
               # Check underscore styled topic

data/lib/karafka/routing/contracts/topic.rb

@@ -66,10 +66,13 @@ module Karafka
 
         virtual do |data, errors|
           next unless errors.empty?
-          next unless ::Karafka::App.config.strict_topics_namespacing
+          next unless Karafka::App.config.strict_topics_namespacing
 
           value = data.fetch(:name)
-          namespacing_chars_count = value.chars.find_all { |c| ['.', '_'].include?(c) }.uniq.size
+          namespace_chars = ['.', '_'].freeze
+          namespacing_chars_count = value.chars.find_all do |c|
+            namespace_chars.include?(c)
+          end.uniq.size
 
           next if namespacing_chars_count <= 1
 

data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb

@@ -39,7 +39,7 @@ module Karafka
               next unless dead_letter_queue[:active]
 
               topic = dead_letter_queue[:topic]
-              topic_regexp = ::Karafka::Contracts::TOPIC_REGEXP
+              topic_regexp = Karafka::Contracts::TOPIC_REGEXP
 
               # When topic is set to false, it means we just want to skip dispatch on DLQ
               next if topic == false

data/lib/karafka/routing/features/declaratives/topic.rb

@@ -16,9 +16,12 @@ module Karafka
           end
 
           # @param active [Boolean] is the topic structure management feature active
-          # @param partitions [Integer]
-          # @param replication_factor [Integer]
+          # @param partitions [Integer] number of partitions for the topic
+          # @param replication_factor [Integer] replication factor for the topic
           # @param details [Hash] extra configuration for the topic
+          # @option details [String] :retention.ms retention time in milliseconds
+          # @option details [String] :compression.type compression type
+          #   (none, gzip, snappy, lz4, zstd)
           # @return [Config] defined structure
           def config(active: true, partitions: 1, replication_factor: 1, **details)
             @declaratives ||= Config.new(

data/lib/karafka/routing/features/deserializers/topic.rb

@@ -22,9 +22,9 @@ module Karafka
           # @param key [Object] deserializer for the message key
           # @param headers [Object] deserializer for the message headers
           def deserializers(
-            payload: ::Karafka::Deserializers::Payload.new,
-            key: ::Karafka::Deserializers::Key.new,
-            headers: ::Karafka::Deserializers::Headers.new
+            payload: Karafka::Deserializers::Payload.new,
+            key: Karafka::Deserializers::Key.new,
+            headers: Karafka::Deserializers::Headers.new
           )
             @deserializers ||= Config.new(
               active: true,

data/lib/karafka/routing/features/inline_insights.rb

@@ -12,11 +12,11 @@ module Karafka
           #
           # @param _config [Karafka::Core::Configurable::Node] app config
           def post_setup(_config)
-            ::Karafka::App.monitor.subscribe('app.running') do
+            Karafka::App.monitor.subscribe('app.running') do
               # Do not activate tracking of statistics if none of our active topics uses it
               # This prevents us from tracking metrics when user just runs a subset of topics
               # in a given process and none of those actually utilizes this feature
-              next unless ::Karafka::App
+              next unless Karafka::App
                           .subscription_groups
                           .values
                           .flat_map(&:itself)
@@ -25,11 +25,11 @@ module Karafka
                           .any?(&:inline_insights?)
 
               # Initialize the tracker prior to becoming multi-threaded
-              ::Karafka::Processing::InlineInsights::Tracker.instance
+              Karafka::Processing::InlineInsights::Tracker.instance
 
               # Subscribe to the statistics reports and collect them
-              ::Karafka.monitor.subscribe(
-                ::Karafka::Processing::InlineInsights::Listener.new
+              Karafka.monitor.subscribe(
+                Karafka::Processing::InlineInsights::Listener.new
               )
             end
           end

data/lib/karafka/routing/router.rb

@@ -8,7 +8,7 @@ module Karafka
     #  structure so all the routes are being stored in a single level array
     module Router
       # Finds first reference of a given topic based on provided lookup attribute
-      # @param lookup [Hash<Symbol, String>] hash with attribute - value key pairs
+      # @param lookup [Hash{Symbol => String}] hash with attribute - value key pairs
       # @return [Karafka::Routing::Topic, nil] proper route details or nil if not found
       def find_by(lookup)
         App.consumer_groups.each do |consumer_group|

data/lib/karafka/routing/subscription_group.rb

@@ -30,7 +30,7 @@ module Karafka
             @group_counter ||= 0
             @group_counter += 1
 
-            ::Digest::SHA256.hexdigest(
+            Digest::SHA256.hexdigest(
               @group_counter.to_s
             )[0..11]
           end

data/lib/karafka/routing/subscription_groups_builder.rb

@@ -24,6 +24,7 @@ module Karafka
 
       private_constant :DISTRIBUTION_KEYS
 
+      # Initializes the subscription groups builder
       def initialize
         @position = -1
       end

data/lib/karafka/routing/topic.rb

@@ -29,7 +29,7 @@ module Karafka
 
       private_constant :INHERITABLE_ATTRIBUTES
 
-      # @param [String, Symbol] name of a topic on which we want to listen
+      # @param name [String, Symbol] name of a topic on which we want to listen
       # @param consumer_group [Karafka::Routing::ConsumerGroup] owning consumer group of this topic
       def initialize(name, consumer_group)
         @name = name.to_s
@@ -92,7 +92,7 @@ module Karafka
           # consumer class is defined with a name. It won't support code reload for anonymous
           # consumer classes, but this is an edge case
           begin
-            ::Object.const_get(@consumer.to_s)
+            Object.const_get(@consumer.to_s)
           rescue NameError
             # It will only fail if the in case of anonymous classes
             @consumer
@@ -139,7 +139,7 @@ module Karafka
           [attribute, public_send(attribute)]
         end
 
-        Hash[map].merge!(
+        map.to_h.merge!(
           id: id,
           name: name,
           active: active?,

data/lib/karafka/server.rb

@@ -174,7 +174,7 @@ module Karafka
           # This ensures that if users have configured the default pool, it is closed correctly
           #
           # Custom pools need to be closed by users themselves
-          ::WaterDrop::ConnectionPool.close
+          WaterDrop::ConnectionPool.close
 
           Karafka::App.terminate!
         end

data/lib/karafka/setup/attributes_map.rb

@@ -122,6 +122,7 @@ module Karafka
         sasl.oauthbearer.config
         sasl.oauthbearer.extensions
         sasl.oauthbearer.grant.type
+        sasl.oauthbearer.metadata.authentication.type
         sasl.oauthbearer.method
         sasl.oauthbearer.scope
         sasl.oauthbearer.token.endpoint.url
@@ -279,6 +280,7 @@ module Karafka
         sasl.oauthbearer.config
         sasl.oauthbearer.extensions
         sasl.oauthbearer.grant.type
+        sasl.oauthbearer.metadata.authentication.type
         sasl.oauthbearer.method
         sasl.oauthbearer.scope
         sasl.oauthbearer.token.endpoint.url
@@ -357,7 +359,7 @@ module Karafka
         end
 
         # @private
-        # @return [Hash<Symbol, Array<Symbol>>] hash with consumer and producer attributes list
+        # @return [Hash{Symbol => Array<Symbol>}] hash with consumer and producer attributes list
         #   that is sorted.
         # @note This method should not be used directly. It is only used to generate appropriate
         #   options list in case it would change
@@ -367,7 +369,7 @@ module Karafka
 
           attributes = { consumer: Set.new, producer: Set.new }
 
-          ::URI.parse(SOURCE).open.readlines.each do |line|
+          URI.parse(SOURCE).open.readlines.each do |line|
             next unless line.include?('|')
 
             attribute, attribute_type = line.split('|').map(&:strip)

data/lib/karafka/setup/config.rb

@@ -12,7 +12,7 @@ module Karafka
     #   enough and will still keep the code simple
     # @see Karafka::Setup::Configurators::Base for more details about configurators api
     class Config
-      extend ::Karafka::Core::Configurable
+      extend Karafka::Core::Configurable
 
       # Available settings
 
@@ -33,9 +33,9 @@ module Karafka
       #   Used only for logging.
       setting :client_id, default: 'karafka'
       # option logger [Instance] logger that we want to use
-      setting :logger, default: ::Karafka::Instrumentation::Logger.new
+      setting :logger, default: Karafka::Instrumentation::Logger.new
       # option monitor [Instance] monitor that we will to use (defaults to Karafka::Monitor)
-      setting :monitor, default: ::Karafka::Instrumentation::Monitor.new
+      setting :monitor, default: Karafka::Instrumentation::Monitor.new
       # option [Boolean] should we reload consumers with each incoming batch thus effectively
       # supporting code reload (if someone reloads code) or should we keep the persistence
       setting :consumer_persistence, default: true
@@ -211,7 +211,7 @@ module Karafka
         setting :cli do
           # option contract [Object] cli setup validation contract (in the context of options and
           # topics)
-          setting :contract, default: ::Karafka::Cli::Contracts::Server.new
+          setting :contract, default: Karafka::Cli::Contracts::Server.new
         end
 
         setting :routing do
@@ -345,7 +345,7 @@ module Karafka
           #   (incoming). Can be replaced with a custom implementation for formats like Avro,
           #   Protobuf, etc. This is a global setting because Rails serializes jobs before
           #   Karafka receives them, so we need a consistent approach across all ActiveJob topics.
-          setting :deserializer, default: ::Karafka::ActiveJob::Deserializer.new
+          setting :deserializer, default: Karafka::ActiveJob::Deserializer.new
         end
       end
 
@@ -415,17 +415,21 @@ module Karafka
           # of the pro defaults with custom components
           Pro::Loader.pre_setup_all(config) if Karafka.pro?
 
-          configure(&)
+          # Wrap config in a proxy that intercepts producer block configuration
+          proxy = ConfigProxy.new(config)
+          # We need to check for the block presence here because user can just run setup without
+          # any block given
+          configure { yield(proxy) if block_given? }
 
           Contracts::Config.new.validate!(
             config.to_h,
             scope: %w[config]
           )
 
-          configure_components
+          configure_components(proxy)
 
           # Refreshes the references that are cached that might have been changed by the config
-          ::Karafka.refresh!
+          Karafka.refresh!
 
           # Post-setup configure all routing features that would need this
           Routing::Features::Base.post_setup_all(config)
@@ -443,14 +447,16 @@ module Karafka
         private
 
         # Sets up all the components that are based on the user configuration
+        # @param config_proxy [ConfigProxy] the configuration proxy containing deferred setup
+        #   blocks
         # @note At the moment it is only WaterDrop
-        def configure_components
+        def configure_components(config_proxy)
           oauth_listener = config.oauth.token_provider_listener
           # We need to subscribe the oauth listener here because we want it to be ready before
           # any consumer/admin runs
           Karafka::App.monitor.subscribe(oauth_listener) if oauth_listener
 
-          config.producer ||= ::WaterDrop::Producer.new do |producer_config|
+          config.producer ||= WaterDrop::Producer.new do |producer_config|
             # In some cases WaterDrop updates the config and we don't want our consumer config to
             # be polluted by those updates, that's why we copy
             producer_kafka = AttributesMap.producer(config.kafka.dup)
@@ -463,6 +469,11 @@ module Karafka
             producer_config.oauth.token_provider_listener = oauth_listener
             producer_config.logger = config.logger
           end
+
+          # Execute user's producer configuration block
+          # This happens after the default producer setup, allowing users to customize settings
+          # If no block was provided during setup, this will be an empty lambda that does nothing
+          config_proxy.producer_initialization_block.call(config.producer.config)
         end
       end
     end