karafka-web 0.11.2 → 0.11.4

data/lib/karafka/web/tracking/consumers/sampler.rb

@@ -15,7 +15,7 @@ module Karafka
           # Current schema version
           # This is used for detecting incompatible changes and not using outdated data during
           # upgrades
-          SCHEMA_VERSION = '1.4.1'
+          SCHEMA_VERSION = '1.5.0'
 
           # Counters that count events occurrences during the given window
           COUNTERS_BASE = {
@@ -72,6 +72,17 @@ module Karafka
             @memory_total_usage = 0
             @memory_usage = 0
             @cpu_usage = [-1, -1, -1]
+
+            # Select and instantiate appropriate system metrics collector based on environment
+            # Use container-aware collector if cgroups are available, otherwise use OS-based
+            metrics_class = if Metrics::Container.active?
+                              Metrics::Container
+                            else
+                              Metrics::Os
+                            end
+            @system_metrics = metrics_class.new(@shell)
+            @network_metrics = Metrics::Network.new(@windows)
+            @server_metrics = Metrics::Server.new
           end
 
           # We cannot report and track the same time, that is why we use mutex here. To make sure
@@ -94,7 +105,7 @@ module Karafka
                 started_at: started_at,
                 status: ::Karafka::App.config.internal.status.to_s,
                 execution_mode: ::Karafka::Server.execution_mode.to_s,
-                listeners: listeners,
+                listeners: @server_metrics.listeners,
                 workers: workers,
                 memory_usage: @memory_usage,
                 memory_total_usage: @memory_total_usage,
@@ -103,8 +114,8 @@ module Karafka
                 threads: threads,
                 cpu_usage: @cpu_usage,
                 tags: Karafka::Process.tags,
-                bytes_received: bytes_received,
-                bytes_sent: bytes_sent
+                bytes_received: @network_metrics.bytes_received,
+                bytes_sent: @network_metrics.bytes_sent
               },
 
               versions: {
@@ -117,8 +128,8 @@ module Karafka
                 librdkafka: librdkafka_version
               },
 
-              stats: jobs_queue_statistics.merge(
-                utilization: utilization
+              stats: jobs_metrics.jobs_queue_statistics.merge(
+                utilization: jobs_metrics.utilization
               ).merge(total: @counters),
 
               consumer_groups: enriched_consumer_groups,
@@ -156,133 +167,44 @@ module Karafka
             @started_at ||= float_now
           end
 
-          # @return [Numeric] % utilization of all the threads. 100% means all the threads are
-          #   utilized all the time within the given time window. 0% means, nothing is happening
-          #   most if not all the time.
-          def utilization
-            totals = windows.m1[:processed_total_time]
-
-            return 0 if totals.empty?
-
-            timefactor = float_now - @started_at
-            timefactor = timefactor > 60 ? 60 : timefactor
-
-            # We divide by 1_000 to convert from milliseconds
-            # We multiply by 100 to have it in % scale
-            (totals.sum / 1_000 / workers / timefactor * 100).round(2)
-          end
-
-          # @return [Hash] number of active and standby listeners
-          def listeners
-            if Karafka::Server.listeners
-              active = Karafka::Server.listeners.count(&:active?)
-              total = Karafka::Server.listeners.count.to_i
-
-              { active: active, standby: total - active }
-            else
-              { active: 0, standby: 0 }
-            end
+          # @return [Metrics::Jobs] jobs metrics instance
+          # @note Lazy initialization since it depends on started_at and workers
+          def jobs_metrics
+            @jobs_metrics ||= Metrics::Jobs.new(@windows, started_at, workers)
           end
 
           # @return [Integer] memory used by this process in kilobytes
           def memory_usage
-            pid = ::Process.pid
-
-            case RUBY_PLATFORM
-            # Reading this that way is cheaper than running a shell command
-            when /linux/
-              IO.readlines("/proc/#{pid}/status").each do |line|
-                next unless line.start_with?('VmRSS:')
-
-                break line.split[1].to_i
-              end
-            when /darwin|bsd/
-              @shell
-                .call("ps -o pid,rss -p #{pid}")
-                .lines
-                .last
-                .split
-                .last
-                .to_i
-            else
-              0
-            end
-          end
-
-          # @return [Hash] job queue statistics
-          def jobs_queue_statistics
-            # We return empty stats in case jobs queue is not yet initialized
-            base = Karafka::Server.jobs_queue&.statistics || { busy: 0, enqueued: 0 }
-            stats = base.slice(:busy, :enqueued, :waiting)
-            stats[:waiting] ||= 0
-            # busy - represents number of jobs that are being executed currently
-            # enqueued - jobs that are in the queue but not being picked up yet
-            # waiting - jobs that are not scheduled on the queue but will be
-            # be enqueued in case of advanced schedulers
-            stats
+            @system_metrics.memory_usage
           end
 
           # Total memory used in the OS
           def memory_total_usage
-            return 0 unless @memory_threads_ps
-
-            @memory_threads_ps.map(&:first).sum
+            @system_metrics.memory_total_usage(@memory_threads_ps)
           end
 
-          # @return [Integer] total amount of memory
+          # @return [Integer] total amount of memory in kilobytes
+          # In containerized environments (Docker/Kubernetes), this returns the container's
+          # memory limit. Otherwise, returns the host's total memory.
           def memory_size
-            @memory_size ||= case RUBY_PLATFORM
-                             when /linux/
-                               mem_info = File.read('/proc/meminfo')
-                               mem_total_line = mem_info.match(/MemTotal:\s*(?<total>\d+)/)
-                               mem_total_line['total'].to_i
-                             when /darwin|bsd/
-                               @shell
-                             .call('sysctl -a')
-                             .split("\n")
-                             .find { |line| line.start_with?('hw.memsize:') }
-                             .to_s
-                             .split(' ')
-                             .last
-                             .to_i
-                             else
-                               0
-                             end
+            @memory_size ||= @system_metrics.memory_size
           end
 
           # @return [Array<Float>] load averages for last 1, 5 and 15 minutes
           def cpu_usage
-            case RUBY_PLATFORM
-            when /linux/
-              File
-                .read('/proc/loadavg')
-                .split(' ')
-                .first(3)
-                .map(&:to_f)
-            when /darwin|bsd/
-              @shell
-                .call('w | head -1')
-                .strip
-                .split(' ')
-                .map(&:to_f)
-                .last(3)
-            else
-              [-1, -1, -1]
-            end
+            @system_metrics.cpu_usage
           end
 
           # @return [Integer] number of process threads.
           # @note This returns total number of threads from the OS perspective including native
           #   extensions threads, etc.
           def threads
-            return 0 unless @memory_threads_ps
-
-            @memory_threads_ps.find { |row| row.last == ::Process.pid }[1]
+            @system_metrics.threads(@memory_threads_ps)
           end
 
           # @return [Integer] CPU count
           def cpus
-            @cpus ||= Etc.nprocessors
+            @cpus ||= @system_metrics.cpus
           end
 
           # @return [Integer] number of threads that process work
@@ -292,119 +214,16 @@ module Karafka
 
           # Loads our ps results into memory so we can extract from them whatever we need
           def memory_threads_ps
-            @memory_threads_ps = case RUBY_PLATFORM
-                                 when /linux/
-                                   page_size = Karafka::Web::Tracking::Helpers::Sysconf.page_size
-                                   status_file = "/proc/#{::Process.pid}/status"
-
-                                   pid = status_file.match(%r{/proc/(\d+)/status})[1]
-
-                                   # Extract thread count from /proc/<pid>/status
-                                   thcount = File.read(status_file)[/^Threads:\s+(\d+)/, 1].to_i
-
-                                   # Extract RSS from /proc/<pid>/statm (second field)
-                                   statm_file = "/proc/#{pid}/statm"
-                                   rss_pages = File.read(statm_file).split[1].to_i rescue 0
-                                   # page size is retrieved from Sysconf
-                                   rss_kb = (rss_pages * page_size) / 1024
-
-                                   [[rss_kb, thcount, pid.to_i]]
-                                 # thcount is not available on macos ps
-                                 # because of that we inject 0 as threads count similar to how
-                                 # we do on windows
-                                 when /darwin|bsd/
-                                   @shell
-                                 .call('ps -A -o rss=,pid=')
-                                 .split("\n")
-                                 .map { |row| row.strip.split(' ').map(&:to_i) }
-                                 .map { |row| [row.first, 0, row.last] }
-                                 else
-                                   @memory_threads_ps = false
-                                 end
+            @memory_threads_ps = @system_metrics.memory_threads_ps
           end
 
           # Consumer group details need to be enriched with details about polling that comes from
           # Karafka level. It is also time based, hence we need to materialize it only at the
           # moment of message dispatch to have it accurate.
           def enriched_consumer_groups
-            @consumer_groups.each_value do |cg_details|
-              cg_details.each do
-                cg_details.fetch(:subscription_groups, {}).each do |sg_id, sg_details|
-                  # This should be always available, since the subscription group polled at time
-                  # is first initialized before we start polling, there should be no case where
-                  # we have statistics about a given subscription group but we do not have the
-                  # sg reference
-                  sg_tracking = subscription_groups.fetch(sg_id)
-
-                  polled_at = sg_tracking.fetch(:polled_at)
-                  sg_details[:state][:poll_age] = (monotonic_now - polled_at).round(2)
-
-                  sg_details[:topics].each do |topic_name, topic_details|
-                    topic_details[:partitions].each do |partition_id, partition_details|
-                      # Always assume non-transactional as default. Will be overwritten by the
-                      # consumer level details if collected
-                      partition_details[:transactional] ||= false
-
-                      # If we have stored offset or stored lag, it means it's not a transactional
-                      # consumer at all so we can skip enrichment
-                      next if partition_details[:lag_stored].positive?
-                      next if partition_details[:stored_offset].positive?
-                      next unless sg_tracking[:topics].key?(topic_name)
-                      next unless sg_tracking[:topics][topic_name].key?(partition_id)
-
-                      k_partition_details = sg_tracking[:topics][topic_name][partition_id]
-
-                      # If seek offset was not yey set, nothing to enrich
-                      next unless k_partition_details[:seek_offset].positive?
-
-                      partition_details[:transactional] = k_partition_details[:transactional]
-
-                      # Seek offset is always +1 from the last stored in Karafka
-                      seek_offset = k_partition_details[:seek_offset]
-                      stored_offset = seek_offset - 1
-
-                      # In case of transactions we have to compute the lag ourselves
-                      # -1 because ls offset (or high watermark) is last + 1
-                      lag = partition_details[:ls_offset] - seek_offset
-                      # This can happen if ls_offset is refreshed slower than our stored offset
-                      # fetching from Karafka transactional layer
-                      lag = 0 if lag.negative?
-
-                      partition_details[:lag] = lag
-                      partition_details[:lag_d] = 0
-                      partition_details[:lag_stored] = lag
-                      partition_details[:lag_stored_d] = 0
-                      partition_details[:stored_offset] = stored_offset
-                      partition_details[:committed_offset] = stored_offset
-                    end
-                  end
-                end
-              end
-            end
-
-            @consumer_groups
-          end
-
-          # @return [Integer] number of bytes received per second out of a one minute time window
-          #   by all the consumers
-          # @note We use one minute window to compensate for cases where metrics would be reported
-          #   or recorded faster or slower. This normalizes data
-          def bytes_received
-            @windows
-              .m1
-              .stats_from { |k, _v| k.end_with?('rxbytes') }
-              .rps
-              .round
-          end
-
-          # @return [Integer] number of bytes sent per second out of a one minute time window by
-          #   all the consumers
-          def bytes_sent
-            @windows
-              .m1
-              .stats_from { |k, _v| k.end_with?('txbytes') }
-              .rps
-              .round
+            Enrichers::ConsumerGroups
+              .new(@consumer_groups, @subscription_groups)
+              .call
           end
         end
       end

data/lib/karafka/web/tracking/contracts/error.rb

@@ -12,6 +12,7 @@ module Karafka
           configure
 
           required(:schema_version) { |val| val.is_a?(String) }
+          required(:id) { |val| val.is_a?(String) && !val.empty? }
           required(:type) { |val| val.is_a?(String) && !val.empty? }
           required(:error_class) { |val| val.is_a?(String) && !val.empty? }
           required(:error_message) { |val| val.is_a?(String) }

data/lib/karafka/web/tracking/helpers/ttls/hash.rb

@@ -18,13 +18,12 @@ module Karafka
             # interested in using for aggregated stats. Once filtered, builds a Stats object out
             # of the candidates
             #
-            # @param block [Proc] block for selection of elements for stats
             # @yieldparam [String] key
             # @yieldparam [Ttls::Array] samples
             # @return [Stats]
-            def stats_from(&block)
+            def stats_from(&)
               Stats.new(
-                select(&block)
+                select(&)
               )
             end
 

data/lib/karafka/web/tracking/helpers/ttls/stats.rb

@@ -17,8 +17,7 @@ module Karafka
             def initialize(ttls_hash)
               @data = ttls_hash
                       .values
-                      .map(&:samples)
-                      .map(&:to_a)
+                      .map { |value| value.samples.to_a }
                       .delete_if { |samples| samples.size < 2 }
                       .map { |samples| samples.map(&:values) }
             end

data/lib/karafka/web/tracking/producers/listeners/base.rb

@@ -12,7 +12,7 @@ module Karafka
             extend Forwardable
 
             def_delegators :sampler, :track
-            def_delegators :reporter, :report, :report!
+            def_delegators :reporter, :report
 
             private
 

data/lib/karafka/web/tracking/producers/listeners/errors.rb

@@ -10,7 +10,7 @@ module Karafka
             include Tracking::Helpers::ErrorInfo
 
             # Schema used by producers error reporting
-            SCHEMA_VERSION = '1.1.0'
+            SCHEMA_VERSION = '1.2.0'
 
             private_constant :SCHEMA_VERSION
 
@@ -34,6 +34,7 @@ module Karafka
 
               {
                 schema_version: SCHEMA_VERSION,
+                id: SecureRandom.uuid,
                 producer_id: event[:producer_id],
                 type: type,
                 error_class: error_class,

data/lib/karafka/web/tracking/ui/errors.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      # Namespace for UI specific tracking
+      module Ui
+        # Listener for tracking and reporting Web UI errors directly to Kafka
+        #
+        # Unlike consumer and producer errors that are collected in samplers and dispatched
+        # periodically, UI errors need to be dispatched immediately and asynchronously from
+        # the web process (Puma/Rack) since there's no background reporter running in the web UI.
+        class Errors
+          include ::Karafka::Core::Helpers::Time
+          include Tracking::Helpers::ErrorInfo
+
+          # Schema used by UI error reporting
+          SCHEMA_VERSION = '1.2.0'
+
+          private_constant :SCHEMA_VERSION
+
+          # Tracks any UI related errors and dispatches them to Kafka
+          #
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_error_occurred(event)
+            # Only process UI errors, ignore all other error types
+            return unless event[:type] == 'web.ui.error'
+
+            error_class, error_message, backtrace = extract_error_info(event[:error])
+
+            error_data = {
+              schema_version: SCHEMA_VERSION,
+              id: SecureRandom.uuid,
+              type: event[:type],
+              error_class: error_class,
+              error_message: error_message,
+              backtrace: backtrace,
+              details: {},
+              occurred_at: float_now,
+              process: {
+                id: process_id
+              }
+            }
+
+            # Validate the error data
+            Tracking::Contracts::Error.new.validate!(error_data)
+
+            # Dispatch error to Kafka asynchronously
+            dispatch(error_data)
+          rescue StandardError => e
+            # If we fail to report an error, log it but don't raise to avoid error loops
+            ::Karafka.logger.error("Failed to report UI error: #{e.message}")
+          end
+
+          private
+
+          # @return [String] unique process identifier
+          def process_id
+            @process_id ||= Tracking::Sampler.new.process_id
+          end
+
+          # Dispatches error to Kafka
+          # @param error_data [Hash] error data to dispatch
+          def dispatch(error_data)
+            ::Karafka::Web.producer.produce_async(
+              topic: ::Karafka::Web.config.topics.errors.name,
+              payload: Zlib::Deflate.deflate(error_data.to_json),
+              key: process_id,
+              headers: { 'zlib' => 'true' }
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/base.rb

@@ -91,13 +91,8 @@ module Karafka
           response.write result.content
         end
 
-        # Display appropriate error specific to a given error type
-        plugin :error_handler, classes: [
-          ::Rdkafka::RdkafkaError,
-          Errors::Ui::NotFoundError,
-          Errors::Ui::ProOnlyError,
-          Errors::Ui::ForbiddenError
-        ] do |e|
+        # Catch all unhandled exceptions, report them to Karafka monitoring, and display error page
+        plugin :error_handler do |e|
           @error = true
 
           case e
@@ -107,9 +102,24 @@ module Karafka
           when Errors::Ui::ForbiddenError
             response.status = 403
             view 'shared/exceptions/not_allowed'
-          else
+          when Errors::Ui::NotFoundError
+            response.status = 404
+            view 'shared/exceptions/not_found'
+          when ::Rdkafka::RdkafkaError
             response.status = 404
             view 'shared/exceptions/not_found'
+          else
+            # Report unhandled errors to Karafka monitoring
+            ::Karafka.monitor.instrument(
+              'error.occurred',
+              error: e,
+              caller: self,
+              type: 'web.ui.error'
+            )
+
+            # For all other unhandled errors, show a generic error page
+            response.status = 500
+            view 'shared/exceptions/unhandled_error'
           end
         end
 
@@ -183,7 +193,7 @@ module Karafka
         def render_response(response)
           response.attributes.each do |key, value|
             instance_variable_set(
-              "@#{key}", value
+              :"@#{key}", value
             )
           end
 

data/lib/karafka/web/ui/controllers/requests/execution_wrapper.rb

@@ -26,12 +26,10 @@ module Karafka
             # Delegates any method call to the controller and wraps it with before/after hooks
             #
             # @param method_name [Symbol] the name of the method being called
-            # @param args [Array] arguments passed to the method
-            # @param block [Proc] optional block passed to the method
             # @return [Object] the result of the delegated controller method for Roda to operate on
-            def method_missing(method_name, *args, &block)
+            def method_missing(method_name, *, &)
               @controller.run_before_hooks(method_name)
-              result = @controller.public_send(method_name, *args, &block)
+              result = @controller.public_send(method_name, *, &)
               @controller.run_after_hooks(method_name)
               result
             end

data/lib/karafka/web/ui/controllers/requests/params.rb

@@ -99,7 +99,7 @@ module Karafka
             def current_offset
               @current_offset ||= begin
                 offset = @request_params.fetch('offset', -1).to_i
-                offset < -1 ? -1 : offset
+                [offset, -1].max
               end
             end
 

data/lib/karafka/web/ui/helpers/application_helper.rb

@@ -247,7 +247,7 @@ module Karafka
               else
                 name = attribute.to_s.tr('_', ' ').tr('?', '')
                 # Always capitalize the name
-                name = name.split(' ').map(&:capitalize).join(' ')
+                name = name.split.map(&:capitalize).join(' ')
               end
             end
 

data/lib/karafka/web/ui/helpers/paths_helper.rb

@@ -87,27 +87,24 @@ module Karafka
 
           # Helps build topics paths
           #
-          # @param args [Array<String>] path params for the topics scope
           # @return [String] topics scope path
-          def topics_path(*args)
-            root_path('topics', *args)
+          def topics_path(*)
+            root_path('topics', *)
           end
 
           # Helps build consumers paths
           #
-          # @param args [Array<String>] path params for consumers scope
           # @return [String] consumers scope path
-          def consumers_path(*args)
-            root_path('consumers', *args)
+          def consumers_path(*)
+            root_path('consumers', *)
           end
 
           # Helps build per-consumer scope paths
           #
           # @param consumer_id [String] consumer process id
-          # @param args [Array<String>] other path components
           # @return [String] per consumer specific path
-          def consumer_path(consumer_id, *args)
-            consumers_path(consumer_id, *args)
+          def consumer_path(consumer_id, *)
+            consumers_path(consumer_id, *)
           end
 
           # Helps build scheduled messages paths.

data/lib/karafka/web/ui/lib/sorter.rb

@@ -22,7 +22,7 @@ module Karafka
           #   we can sort on method invocations, this needs to be limited and provided on a per
           #   controller basis.
           def initialize(sort_query, allowed_attributes:)
-            field, order = sort_query.split(' ')
+            field, order = sort_query.split
 
             @order = order.to_s.downcase
             @order = ALLOWED_ORDERS.first unless ALLOWED_ORDERS.include?(@order)

data/lib/karafka/web/ui/models/health.rb

@@ -57,10 +57,15 @@ module Karafka
                 pt_id = partition.id
 
                 stats[cg_id] ||= { topics: {} }
-                stats[cg_id][:topics][t_name] ||= {}
-                stats[cg_id][:topics][t_name][pt_id] = partition
-                stats[cg_id][:topics][t_name][pt_id][:process] = process
-                stats[cg_id][:topics][t_name][pt_id][:subscription_group_id] = sg_id
+
+                stats[cg_id][:topics][t_name] ||= {
+                  partitions: {},
+                  partitions_count: topic.partitions_cnt
+                }
+
+                stats[cg_id][:topics][t_name][:partitions][pt_id] = partition
+                stats[cg_id][:topics][t_name][:partitions][pt_id][:process] = process
+                stats[cg_id][:topics][t_name][:partitions][pt_id][:subscription_group_id] = sg_id
               end
             end
 
@@ -74,7 +79,7 @@ module Karafka
 
                 ages = consumer_group[:subscription_groups].values.map do |sub_group_details|
                   rebalance_age_ms = sub_group_details[:state][:rebalance_age] || 0
-                  dispatched_at - rebalance_age_ms / 1_000
+                  dispatched_at - (rebalance_age_ms / 1_000)
                 end
 
                 stats[cg_name][:rebalance_ages] ||= Set.new
@@ -119,15 +124,15 @@ module Karafka
               stats.each_value do |cg_data|
                 topics = cg_data[:topics]
 
-                topics.each do |topic_name, t_data|
-                  topics[topic_name] = Hash[t_data.sort_by { |key, _| key }]
+                topics.each_value do |t_data|
+                  t_data[:partitions] = t_data[:partitions].sort_by { |key, _| key }.to_h
                 end
 
-                cg_data[:topics] = Hash[topics.sort_by { |key, _| key }]
+                cg_data[:topics] = topics.sort_by { |key, _| key }.to_h
               end
 
               # Ensure that all consumer groups are always in the same order
-              Hash[stats.sort_by { |key, _| key }]
+              stats.sort_by { |key, _| key }.to_h
             end
           end
         end

data/lib/karafka/web/ui/models/jobs.rb

@@ -30,16 +30,14 @@ module Karafka
           end
 
           # Creates a new Jobs object with selected jobs
-          # @param block [Proc] select proc
           # @return [Jobs] selected jobs enclosed with the Jobs object
-          def select(&block)
-            self.class.new(super(&block))
+          def select(&)
+            self.class.new(super)
           end
 
           # Allows for iteration over jobs
-          # @param block [Proc] block to call for each job
-          def each(&block)
-            @jobs_array.each(&block)
+          def each(&)
+            @jobs_array.each(&)
           end
         end
       end