karafka 2.4.18 → 2.5.0

data/lib/karafka/connection/listener.rb

@@ -12,6 +12,13 @@ module Karafka
     class Listener
       include Helpers::Async
 
+      include Helpers::ConfigImporter.new(
+        jobs_builder: %i[internal processing jobs_builder],
+        partitioner_class: %i[internal processing partitioner_class],
+        reset_backoff: %i[internal connection reset_backoff],
+        listener_thread_priority: %i[internal connection listener_thread_priority]
+      )
+
       # Can be useful for logging
       # @return [String] id of this listener
       attr_reader :id
@@ -19,6 +26,11 @@ module Karafka
       # @return [Karafka::Routing::SubscriptionGroup] subscription group that this listener handles
       attr_reader :subscription_group
 
+      # @return [Processing::CoordinatorsBuffer] coordinator buffers that can be used directly in
+      #   advanced cases of changes to the polling flow (like triggered seek back without messages
+      #   ahead in the topic)
+      attr_reader :coordinators
+
       # How long to wait in the initial events poll. Increases chances of having the initial events
       # immediately available
       INITIAL_EVENTS_POLL_TIMEOUT = 100
@@ -30,16 +42,13 @@ module Karafka
       # @param scheduler [Karafka::Processing::Scheduler] scheduler we want to use
       # @return [Karafka::Connection::Listener] listener instance
       def initialize(subscription_group, jobs_queue, scheduler)
-        proc_config = ::Karafka::App.config.internal.processing
-
         @id = SecureRandom.hex(6)
         @subscription_group = subscription_group
         @jobs_queue = jobs_queue
         @coordinators = Processing::CoordinatorsBuffer.new(subscription_group.topics)
         @client = Client.new(@subscription_group, -> { running? })
         @executors = Processing::ExecutorsBuffer.new(@client, subscription_group)
-        @jobs_builder = proc_config.jobs_builder
-        @partitioner = proc_config.partitioner_class.new(subscription_group)
+        @partitioner = partitioner_class.new(subscription_group)
         @scheduler = scheduler
         @events_poller = Helpers::IntervalRunner.new { @client.events_poll }
         # We keep one buffer for messages to preserve memory and not allocate extra objects
@@ -111,7 +120,10 @@ module Karafka
 
         @status.start!
 
-        async_call("karafka.listener##{@subscription_group.id}")
+        async_call(
+          "karafka.listener##{@subscription_group.id}",
+          listener_thread_priority
+        )
       end
 
       # Stops the jobs queue, triggers shutdown on all the executors (sync), commits offsets and
@@ -254,7 +266,7 @@ module Karafka
         reset
 
         # Ruby sleep is in seconds
-        sleep_time = ::Karafka::App.config.internal.connection.reset_backoff / 10_000.0
+        sleep_time = reset_backoff / 10_000.0
         sleep(sleep_time) && retry
       end
 
@@ -294,7 +306,7 @@ module Karafka
             # here. In cases like this, we do not run a revocation job
             @executors.find_all(topic, partition).each do |executor|
               executor.coordinator.increment(:revoked)
-              jobs << @jobs_builder.revoked(executor)
+              jobs << jobs_builder.revoked(executor)
             end
 
             # We need to remove all the executors of a given topic partition that we have lost, so
@@ -318,7 +330,7 @@ module Karafka
 
         @executors.each do |executor|
           executor.coordinator.increment(:shutdown)
-          job = @jobs_builder.shutdown(executor)
+          job = jobs_builder.shutdown(executor)
           jobs << job
         end
 
@@ -355,7 +367,7 @@ module Karafka
             if coordinator.topic.eofed?
               @executors.find_all_or_create(topic, partition, coordinator).each do |executor|
                 coordinator.increment(:eofed)
-                eofed_jobs << @jobs_builder.eofed(executor)
+                eofed_jobs << jobs_builder.eofed(executor)
               end
             end
 
@@ -372,7 +384,7 @@ module Karafka
             # Start work coordination for this topic partition
             coordinator.increment(:idle)
             executor = @executors.find_or_create(topic, partition, 0, coordinator)
-            idle_jobs << @jobs_builder.idle(executor)
+            idle_jobs << jobs_builder.idle(executor)
 
             next
           end
@@ -383,7 +395,7 @@ module Karafka
           @partitioner.call(topic, messages, coordinator) do |group_id, partition_messages|
             coordinator.increment(:consume)
             executor = @executors.find_or_create(topic, partition, group_id, coordinator)
-            consume_jobs << @jobs_builder.consume(executor, partition_messages)
+            consume_jobs << jobs_builder.consume(executor, partition_messages)
           end
         end
 
@@ -451,7 +463,7 @@ module Karafka
 
             @executors.find_all_or_create(topic_name, partition, coordinator).each do |executor|
               coordinator.increment(:periodic)
-              jobs << @jobs_builder.periodic(executor)
+              jobs << jobs_builder.periodic(executor)
             end
           end
         end

data/lib/karafka/connection/messages_buffer.rb

@@ -45,7 +45,7 @@ module Karafka
         last_polled_at = raw_messages_buffer.last_polled_at
 
         raw_messages_buffer.each do |topic, partition, messages, eof|
-          @size += messages.count
+          @size += messages.size
 
           ktopic = @subscription_group.topics.find(topic)
 

data/lib/karafka/connection/proxy.rb

@@ -44,7 +44,7 @@ module Karafka
       # clusters can handle our requests.
       #
       # @param topic [String] topic name
-      # @param partition [Partition]
+      # @param partition [Integer] partition number
       # @return [Array<Integer, Integer>] watermark offsets
       def query_watermark_offsets(topic, partition)
         l_config = @config.query_watermark_offsets
@@ -108,6 +108,7 @@ module Karafka
       rescue Rdkafka::RdkafkaError => e
         return false if e.code == :assignment_lost
         return false if e.code == :state
+        return false if e.code == :illegal_generation
 
         raise e
       end
@@ -136,6 +137,8 @@ module Karafka
           return false
         when :unknown_member_id
           return false
+        when :illegal_generation
+          return false
         when :no_offset
           return true
         when :coordinator_load_in_progress

data/lib/karafka/constraints.rb

@@ -15,13 +15,13 @@ module Karafka
         # Skip verification if web is not used at all
         return unless require_version('karafka/web')
 
-        # All good if version higher than 0.9.0.rc3 because we expect 0.9.0.rc3 or higher
-        return if version(Karafka::Web::VERSION) >= version('0.9.0.rc3')
+        # All good if version higher than 0.10.0 because we expect 0.10.0 or higher
+        return if version(Karafka::Web::VERSION) >= version('0.10.0')
 
         # If older web-ui used, we cannot allow it
         raise(
           Errors::DependencyConstraintsError,
-          'karafka-web < 0.9.0 is not compatible with this karafka version'
+          'karafka-web < 0.10.0 is not compatible with this karafka version'
         )
       end
 

data/lib/karafka/contracts/base.rb

@@ -5,12 +5,13 @@ module Karafka
     # Base contract for all Karafka contracts
     class Base < ::Karafka::Core::Contractable::Contract
       # @param data [Hash] data for validation
+      # @param scope [Array<String>] nested scope if in use
       # @return [Boolean] true if all good
       # @raise [Errors::InvalidConfigurationError] invalid configuration error
       # @note We use contracts only in the config validation context, so no need to add support
       #   for multiple error classes. It will be added when it will be needed.
-      def validate!(data)
-        super(data, Errors::InvalidConfigurationError)
+      def validate!(data, scope: [])
+        super(data, Errors::InvalidConfigurationError, scope: scope)
       end
     end
   end

data/lib/karafka/contracts/config.rb

@@ -35,6 +35,7 @@ module Karafka
       required(:group_id) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
       required(:kafka) { |val| val.is_a?(Hash) && !val.empty? }
       required(:strict_declarative_topics) { |val| [true, false].include?(val) }
+      required(:worker_thread_priority) { |val| (-3..3).to_a.include?(val) }
 
       nested(:swarm) do
         required(:nodes) { |val| val.is_a?(Integer) && val.positive? }
@@ -52,7 +53,8 @@ module Karafka
         required(:kafka) { |val| val.is_a?(Hash) }
         required(:group_id) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
         required(:max_wait_time) { |val| val.is_a?(Integer) && val.positive? }
-        required(:max_attempts) { |val| val.is_a?(Integer) && val.positive? }
+        required(:retry_backoff) { |val| val.is_a?(Integer) && val >= 100 }
+        required(:max_retries_duration) { |val| val.is_a?(Integer) && val >= 1_000 }
       end
 
       # We validate internals just to be sure, that they are present and working
@@ -81,6 +83,7 @@ module Karafka
           required(:manager) { |val| !val.nil? }
           required(:conductor) { |val| !val.nil? }
           required(:reset_backoff) { |val| val.is_a?(Integer) && val >= 1_000 }
+          required(:listener_thread_priority) { |val| (-3..3).to_a.include?(val) }
 
           nested(:proxy) do
             nested(:commit) do
@@ -114,6 +117,7 @@ module Karafka
           required(:jobs_queue_class) { |val| !val.nil? }
           required(:scheduler_class) { |val| !val.nil? }
           required(:coordinator_class) { |val| !val.nil? }
+          required(:errors_tracker_class) { |val| val.nil? || val.is_a?(Class) }
           required(:partitioner_class) { |val| !val.nil? }
           required(:strategy_selector) { |val| !val.nil? }
           required(:expansions_selector) { |val| !val.nil? }

data/lib/karafka/contracts/topic.rb

@@ -70,7 +70,7 @@ module Karafka
         next unless ::Karafka::App.config.strict_topics_namespacing
 
         value = data.fetch(:name)
-        namespacing_chars_count = value.chars.find_all { |c| ['.', '_'].include?(c) }.uniq.count
+        namespacing_chars_count = value.chars.find_all { |c| ['.', '_'].include?(c) }.uniq.size
 
         next if namespacing_chars_count <= 1
 

data/lib/karafka/errors.rb

@@ -22,7 +22,34 @@ module Karafka
     InvalidConfigurationError = Class.new(BaseError)
 
     # Raised when we try to use Karafka CLI commands (except install) without a boot file
-    MissingBootFileError = Class.new(BaseError)
+    MissingBootFileError = Class.new(BaseError) do
+      # @param boot_file_path [Pathname] path where the boot file should be
+      def initialize(boot_file_path)
+        message = <<~MSG
+
+          \e[31mKarafka Boot File Missing:\e[0m #{boot_file_path}
+
+          Cannot find Karafka boot file - this file configures your Karafka application.
+
+          \e[33mQuick fixes:\e[0m
+            \e[32m1.\e[0m Navigate to your Karafka app directory
+            \e[32m2.\e[0m Check if following file exists: \e[36m#{boot_file_path}\e[0m
+            \e[32m3.\e[0m Install Karafka if needed: \e[36mkarafka install\e[0m
+
+          \e[33mCommon causes:\e[0m
+            \e[31m•\e[0m Wrong directory (not in Karafka app root)
+            \e[31m•\e[0m File was accidentally moved or deleted
+            \e[31m•\e[0m New project needing initialization
+
+          For setup help: \e[34mhttps://karafka.io/docs/Getting-Started\e[0m
+        MSG
+
+        super(message)
+        # In case of this error backtrace is irrelevant and we want to print comprehensive error
+        # message without backtrace, this is why nullified.
+        set_backtrace([])
+      end
+    end
 
     # Raised when we've waited enough for shutting down a non-responsive process
     ForcefulShutdownError = Class.new(BaseError)
@@ -35,6 +62,9 @@ module Karafka
     # Raised when given topic is not found while expected
     TopicNotFoundError = Class.new(BaseError)
 
+    # Raised when given consumer group is not found while expected
+    ConsumerGroupNotFoundError = Class.new(BaseError)
+
     # This should never happen. Please open an issue if it does.
     UnsupportedCaseError = Class.new(BaseError)
 
@@ -62,7 +92,17 @@ module Karafka
     ResultNotVisibleError = Class.new(BaseError)
 
     # Raised when there is an attempt to run an unrecognized CLI command
-    UnrecognizedCommandError = Class.new(BaseError)
+    UnrecognizedCommandError = Class.new(BaseError) do
+      # Overwritten not to print backtrace for unknown CLI command
+      def initialize(*args)
+        super
+        set_backtrace([])
+      end
+    end
+
+    # Raised when you were executing a command and it could not finish successfully because of
+    # a setup state or parameters configuration
+    CommandValidationError = Class.new(BaseError)
 
     # Raised when we attempt to perform operation that is only allowed inside of a transaction and
     # there is no transaction around us
@@ -71,6 +111,10 @@ module Karafka
     # Raised in case user would want to perform nested transactions.
     TransactionAlreadyInitializedError = Class.new(BaseError)
 
+    # Raised when user used transactional offset marking but after that tried to use
+    # non-transactional marking, effectively mixing both. This is not allowed.
+    NonTransactionalMarkingAttemptError = Class.new(BaseError)
+
     # Raised in case a listener that was paused is being resumed
     InvalidListenerResumeError = Class.new(BaseError)
 

data/lib/karafka/helpers/async.rb

@@ -37,12 +37,14 @@ module Karafka
 
       # Runs the `#call` method in a new thread
       # @param thread_name [String] name that we want to assign to the thread when we start it
-      def async_call(thread_name)
+      # @param thread_priority [Integer] Ruby thread priority
+      def async_call(thread_name, thread_priority = 0)
         MUTEX.synchronize do
           return if @thread&.alive?
 
           @thread = Thread.new do
             Thread.current.name = thread_name
+            Thread.current.priority = thread_priority
 
             Thread.current.abort_on_exception = true
 

data/lib/karafka/helpers/interval_runner.rb

@@ -30,6 +30,14 @@ module Karafka
         @block.call
       end
 
+      # Runs the requested code bypassing any time frequencies
+      # Useful when we have certain actions that usually need to run periodically but in some
+      # cases need to run asap
+      def call!
+        @last_called_at = monotonic_now
+        @block.call
+      end
+
       # Resets the runner, so next `#call` will run the underlying code
       def reset
         @last_called_at = monotonic_now - @interval

data/lib/karafka/instrumentation/callbacks/rebalance.rb

@@ -12,8 +12,10 @@ module Karafka
 
         # @param subscription_group [Karafka::Routes::SubscriptionGroup] subscription group for
         #   which we want to manage rebalances
-        def initialize(subscription_group)
+        # @param client_id [String] id of the client managing this rebalance
+        def initialize(subscription_group, client_id)
           @subscription_group = subscription_group
+          @client_id = client_id
         end
 
         # Publishes an event that partitions are going to be revoked.
@@ -62,6 +64,7 @@ module Karafka
             subscription_group: @subscription_group,
             consumer_group_id: @subscription_group.consumer_group.id,
             consumer_group: @subscription_group.consumer_group,
+            client_id: @client_id,
             tpl: tpl
           )
         rescue StandardError => e
@@ -71,6 +74,7 @@ module Karafka
             subscription_group_id: @subscription_group.id,
             consumer_group_id: @subscription_group.consumer_group.id,
             type: "callbacks.rebalance.#{name}.error",
+            client_id: @client_id,
             error: e
           )
         end

data/lib/karafka/instrumentation/logger_listener.rb

@@ -24,14 +24,29 @@ module Karafka
         @log_polling = log_polling
       end
 
+      #
+      #
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_connection_listener_before_fetch_loop(event)
+        listener_id = event[:caller].id
+        subscription_group = event[:subscription_group]
+        consumer_group_id = subscription_group.consumer_group.id
+        topics = subscription_group.topics.select(&:active?).map(&:name).join(', ')
+        group_details = "#{consumer_group_id}/#{subscription_group.id}"
+
+        info(
+          "[#{listener_id}] Group #{group_details} subscribing to topics: #{topics}"
+        )
+      end
+
       # Logs each messages fetching attempt
       #
       # @param event [Karafka::Core::Monitoring::Event] event details including payload
       def on_connection_listener_fetch_loop(event)
         return unless log_polling?
 
-        listener = event[:caller]
-        debug "[#{listener.id}] Polling messages..."
+        listener_id = event[:caller].id
+        debug "[#{listener_id}] Polling messages..."
       end
 
       # Logs about messages that we've received from Kafka
@@ -40,11 +55,11 @@ module Karafka
       def on_connection_listener_fetch_loop_received(event)
         return unless log_polling?
 
-        listener = event[:caller]
+        listener_id = event[:caller].id
         time = event[:time].round(2)
         messages_count = event[:messages_buffer].size
 
-        message = "[#{listener.id}] Polled #{messages_count} messages in #{time}ms"
+        message = "[#{listener_id}] Polled #{messages_count} messages in #{time}ms"
 
         # We don't want the "polled 0" in dev as it would spam the log
         # Instead we publish only info when there was anything we could poll and fail over to the
@@ -61,7 +76,7 @@ module Karafka
         consumer = job.executor.topic.consumer
         topic = job.executor.topic.name
         partition = job.executor.partition
-        info "[#{job.id}] #{job_type} job for #{consumer} on #{topic}/#{partition} started"
+        info "[#{job.id}] #{job_type} job for #{consumer} on #{topic}-#{partition} started"
       end
 
       # Prints info about the fact that a given job has finished
@@ -76,7 +91,7 @@ module Karafka
         partition = job.executor.partition
         info <<~MSG.tr("\n", ' ').strip!
           [#{job.id}] #{job_type} job for #{consumer}
-          on #{topic}/#{partition} finished in #{time} ms
+          on #{topic}-#{partition} finished in #{time} ms
         MSG
       end
 
@@ -93,7 +108,7 @@ module Karafka
 
         info <<~MSG.tr("\n", ' ').strip!
           [#{client.id}]
-          Pausing on topic #{topic}/#{partition}
+          Pausing on topic #{topic}-#{partition}
           on #{offset ? "offset #{offset}" : 'the consecutive offset'}
         MSG
       end
@@ -107,7 +122,7 @@ module Karafka
         client = event[:caller]
 
         info <<~MSG.tr("\n", ' ').strip!
-          [#{client.id}] Resuming on topic #{topic}/#{partition}
+          [#{client.id}] Resuming on topic #{topic}-#{partition}
         MSG
       end
 
@@ -123,7 +138,7 @@ module Karafka
 
         info <<~MSG.tr("\n", ' ').strip!
           [#{consumer.id}] Retrying of #{consumer.class} after #{timeout} ms
-          on topic #{topic}/#{partition} from offset #{offset}
+          on topic #{topic}-#{partition} from offset #{offset}
         MSG
       end
 
@@ -138,7 +153,7 @@ module Karafka
 
         info <<~MSG.tr("\n", ' ').strip!
           [#{consumer.id}] Seeking from #{consumer.class}
-          on topic #{topic}/#{partition} to offset #{seek_offset}
+          on topic #{topic}-#{partition} to offset #{seek_offset}
         MSG
       end
 
@@ -147,7 +162,8 @@ module Karafka
       #
       # @param event [Karafka::Core::Monitoring::Event] event details including payload
       def on_process_notice_signal(event)
-        info "Received #{event[:signal]} system signal"
+        server_id = Karafka::Server.id
+        info "[#{server_id}] Received #{event[:signal]} system signal"
 
         # We print backtrace only for ttin
         return unless event[:signal] == :SIGTTIN
@@ -168,38 +184,76 @@ module Karafka
 
       # Logs info that we're running Karafka app.
       #
-      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
-      def on_app_running(_event)
-        info "Running in #{RUBY_DESCRIPTION}"
-        info "Running Karafka #{Karafka::VERSION} server"
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_app_running(event)
+        server_id = event[:server_id]
+
+        info "[#{server_id}] Running in #{RUBY_DESCRIPTION}"
+        info "[#{server_id}] Running Karafka #{Karafka::VERSION} server"
 
         return if Karafka.pro?
 
-        info 'See LICENSE and the LGPL-3.0 for licensing details'
+        info "[#{server_id}] See LICENSE and the LGPL-3.0 for licensing details"
       end
 
-      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
-      def on_app_quieting(_event)
-        info 'Switching to quiet mode. New messages will not be processed'
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_app_quieting(event)
+        info "[#{event[:server_id]}] Switching to quiet mode. New messages will not be processed"
       end
 
-      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
-      def on_app_quiet(_event)
-        info 'Reached quiet mode. No messages will be processed anymore'
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_app_quiet(event)
+        info "[#{event[:server_id]}] Reached quiet mode. No messages will be processed anymore"
       end
 
       # Logs info that we're going to stop the Karafka server.
       #
-      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
-      def on_app_stopping(_event)
-        info 'Stopping Karafka server'
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_app_stopping(event)
+        info "[#{event[:server_id]}] Stopping Karafka server"
       end
 
       # Logs info that we stopped the Karafka server.
       #
-      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
-      def on_app_stopped(_event)
-        info 'Stopped Karafka server'
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_app_stopped(event)
+        info "[#{event[:server_id]}] Stopped Karafka server"
+      end
+
+      # Logs info about partitions we have lost
+      #
+      # @param event [Karafka::Core::Monitoring::Event] event details with revoked partitions
+      def on_rebalance_partitions_revoked(event)
+        revoked_partitions = event[:tpl].to_h.transform_values { |part| part.map(&:partition) }
+        group_id = event[:consumer_group_id]
+        client_id = event[:client_id]
+        group_prefix = "[#{client_id}] Group #{group_id} rebalance"
+
+        if revoked_partitions.empty?
+          info "#{group_prefix}: No partitions revoked"
+        else
+          revoked_partitions.each do |topic, partitions|
+            info "#{group_prefix}: #{topic}-[#{partitions.join(',')}] revoked"
+          end
+        end
+      end
+
+      # Logs info about partitions that we've gained
+      #
+      # @param event [Karafka::Core::Monitoring::Event] event details with assigned partitions
+      def on_rebalance_partitions_assigned(event)
+        assigned_partitions = event[:tpl].to_h.transform_values { |part| part.map(&:partition) }
+        group_id = event[:consumer_group_id]
+        client_id = event[:client_id]
+        group_prefix = "[#{client_id}] Group #{group_id} rebalance"
+
+        if assigned_partitions.empty?
+          info "#{group_prefix}: No partitions assigned"
+        else
+          assigned_partitions.each do |topic, partitions|
+            info "#{group_prefix}: #{topic}-[#{partitions.join(',')}] assigned"
+          end
+        end
       end
 
       # Logs info when we have dispatched a message the the DLQ
@@ -215,7 +269,7 @@ module Karafka
 
         info <<~MSG.tr("\n", ' ').strip!
           [#{consumer.id}] Dispatched message #{offset}
-          from #{topic}/#{partition}
+          from #{topic}-#{partition}
           to DLQ topic: #{dlq_topic}
         MSG
       end
@@ -234,7 +288,7 @@ module Karafka
         info <<~MSG.tr("\n", ' ').strip!
           [#{consumer.id}] Throttled and will resume
           from message #{offset}
-          on #{topic}/#{partition}
+          on #{topic}-#{partition}
         MSG
       end
 
@@ -249,7 +303,7 @@ module Karafka
 
         info <<~MSG.tr("\n", ' ').strip!
           [#{consumer.id}] Post-filtering seeking to message #{offset}
-          on #{topic}/#{partition}
+          on #{topic}-#{partition}
         MSG
       end
 
@@ -371,9 +425,18 @@ module Karafka
         when 'connection.client.unsubscribe.error'
           error "Client unsubscribe error occurred: #{error}"
           error details
+        when 'parallel_segments.reducer.error'
+          error "Parallel segments reducer error occurred: #{error}"
+          error details
+        when 'parallel_segments.partitioner.error'
+          error "Parallel segments partitioner error occurred: #{error}"
+          error details
+        when 'virtual_partitions.partitioner.error'
+          error "Virtual partitions partitioner error occurred: #{error}"
+          error details
         # This handles any custom errors coming from places like Web-UI, etc
         else
-          error "#{type} error occurred: #{error}"
+          error "#{type} error occurred: #{error.class} - #{error}"
           error details
         end
       end

data/lib/karafka/instrumentation/proctitle_listener.rb

@@ -4,6 +4,10 @@ module Karafka
   module Instrumentation
     # Listener that sets a proc title with a nice descriptive value
     class ProctitleListener
+      include Helpers::ConfigImporter.new(
+        client_id: %i[client_id]
+      )
+
       Status::STATES.each_key do |state|
         class_eval <<~RUBY, __FILE__, __LINE__ + 1
           # Updates proc title to an appropriate state
@@ -19,7 +23,7 @@ module Karafka
       # @param status [String] any status we want to set
       def setproctitle(status)
         ::Process.setproctitle(
-          "karafka #{Karafka::App.config.client_id} (#{status})"
+          "karafka #{client_id} (#{status})"
         )
       end
     end

data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb

@@ -131,11 +131,11 @@ module Karafka
             tags = consumer_tags(consumer)
             tags.concat(default_tags)
 
-            count('consumer.messages', messages.count, tags: tags)
+            count('consumer.messages', messages.size, tags: tags)
             count('consumer.batches', 1, tags: tags)
             gauge('consumer.offset', metadata.last_offset, tags: tags)
             histogram('consumer.consumed.time_taken', event[:time], tags: tags)
-            histogram('consumer.batch_size', messages.count, tags: tags)
+            histogram('consumer.batch_size', messages.size, tags: tags)
             histogram('consumer.processing_lag', metadata.processing_lag, tags: tags)
             histogram('consumer.consumption_lag', metadata.consumption_lag, tags: tags)
           end