karafka-web 0.11.3 → 0.11.4

data/lib/karafka/web/tracking/consumers/sampler/enrichers/consumer_groups.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Consumers
+        class Sampler < Tracking::Sampler
+          module Enrichers
+            # Enriches consumer groups data with polling details and transactional consumer handling
+            # This is responsible for materializing time-based data and filling statistical gaps
+            # for transactional consumers
+            class ConsumerGroups < Base
+              include ::Karafka::Core::Helpers::Time
+
+              # @param consumer_groups [Hash] consumer groups hash to be enriched
+              # @param subscription_groups [Hash] subscription groups tracking data
+              def initialize(consumer_groups, subscription_groups)
+                super()
+                @consumer_groups = consumer_groups
+                @subscription_groups = subscription_groups
+              end
+
+              # Enriches consumer groups with polling details and transactional consumer offsets
+              # @return [Hash] enriched consumer groups
+              def call
+                consumer_groups.each_value do |cg_details|
+                  cg_details.each do
+                    cg_details.fetch(:subscription_groups, {}).each do |sg_id, sg_details|
+                      enrich_subscription_group(sg_id, sg_details)
+                    end
+                  end
+                end
+
+                consumer_groups
+              end
+
+              private
+
+              attr_reader :consumer_groups, :subscription_groups
+
+              # Enriches a single subscription group with polling age and partition details
+              # @param sg_id [String] subscription group id
+              # @param sg_details [Hash] subscription group details from statistics
+              def enrich_subscription_group(sg_id, sg_details)
+                # This should be always available, since the subscription group polled at time
+                # is first initialized before we start polling, there should be no case where
+                # we have statistics about a given subscription group but we do not have the
+                # sg reference
+                sg_tracking = subscription_groups.fetch(sg_id)
+
+                polled_at = sg_tracking.fetch(:polled_at)
+                sg_details[:state][:poll_age] = (monotonic_now - polled_at).round(2)
+
+                sg_details[:topics].each do |topic_name, topic_details|
+                  topic_details[:partitions].each do |partition_id, partition_details|
+                    enrich_partition(sg_tracking, topic_name, partition_id, partition_details)
+                  end
+                end
+              end
+
+              # Enriches partition details for transactional consumers
+              # @param sg_tracking [Hash] subscription group tracking data
+              # @param topic_name [String] topic name
+              # @param partition_id [Integer] partition id
+              # @param partition_details [Hash] partition details from statistics
+              def enrich_partition(sg_tracking, topic_name, partition_id, partition_details)
+                # Always assume non-transactional as default. Will be overwritten by the
+                # consumer level details if collected
+                partition_details[:transactional] ||= false
+
+                # If we have stored offset or stored lag, it means it's not a transactional
+                # consumer at all so we can skip enrichment
+                return if partition_details[:lag_stored].positive?
+                return if partition_details[:stored_offset].positive?
+                return unless sg_tracking[:topics].key?(topic_name)
+                return unless sg_tracking[:topics][topic_name].key?(partition_id)
+
+                k_partition_details = sg_tracking[:topics][topic_name][partition_id]
+
+                # If seek offset was not yet set, nothing to enrich
+                return unless k_partition_details[:seek_offset].positive?
+
+                enrich_transactional_partition(k_partition_details, partition_details)
+              end
+
+              # Enriches partition with transactional consumer offset details
+              # @param k_partition_details [Hash] Karafka-level partition details
+              # @param partition_details [Hash] partition details from statistics
+              def enrich_transactional_partition(k_partition_details, partition_details)
+                partition_details[:transactional] = k_partition_details[:transactional]
+
+                # Seek offset is always +1 from the last stored in Karafka
+                seek_offset = k_partition_details[:seek_offset]
+                stored_offset = seek_offset - 1
+
+                # In case of transactions we have to compute the lag ourselves
+                # -1 because ls offset (or high watermark) is last + 1
+                lag = partition_details[:ls_offset] - seek_offset
+                # This can happen if ls_offset is refreshed slower than our stored offset
+                # fetching from Karafka transactional layer
+                lag = 0 if lag.negative?
+
+                partition_details[:lag] = lag
+                partition_details[:lag_d] = 0
+                partition_details[:lag_stored] = lag
+                partition_details[:lag_stored_d] = 0
+                partition_details[:stored_offset] = stored_offset
+                partition_details[:committed_offset] = stored_offset
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/consumers/sampler/metrics/base.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Consumers
+        class Sampler < Tracking::Sampler
+          # Namespace for metrics collectors that gather various system and process statistics
+          module Metrics
+            # Base class for metrics collectors
+            # This is an abstract base class that can be extended to create custom metrics collectors
+            class Base
+              # Placeholder for future common functionality
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/consumers/sampler/metrics/container.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Consumers
+        class Sampler < Tracking::Sampler
+          module Metrics
+            # Collects metrics from containerized environments (Docker, Kubernetes)
+            # Extends OS metrics with container-aware memory limit detection from cgroups
+            # Supports both cgroups v1 and v2
+            class Container < Os
+              # Maximum value that represents "no limit" in cgroup v2
+              CGROUP_V2_MAX = 'max'
+
+              # Paths for cgroup detection and reading
+              CGROUP_V2_CONTROLLERS = '/sys/fs/cgroup/cgroup.controllers'
+
+              # Memory paths
+              # Path to cgroup v1 memory limit file
+              CGROUP_V1_MEMORY_LIMIT = '/sys/fs/cgroup/memory/memory.limit_in_bytes'
+              # Path to cgroup v2 memory limit file
+              CGROUP_V2_MEMORY_LIMIT = '/sys/fs/cgroup/memory.max'
+
+              private_constant(
+                :CGROUP_V2_MAX,
+                :CGROUP_V2_CONTROLLERS,
+                :CGROUP_V1_MEMORY_LIMIT,
+                :CGROUP_V2_MEMORY_LIMIT
+              )
+
+              class << self
+                # Checks if running in a containerized environment with cgroups
+                # @return [Boolean] true if cgroups are available, false otherwise
+                def active?
+                  !cgroup_version.nil?
+                end
+
+                # Gets the memory limit for the container
+                # @return [Integer, nil] memory limit in kilobytes, or nil if not available
+                def memory_limit
+                  return @memory_limit if instance_variable_defined?(:@memory_limit)
+
+                  @memory_limit = case cgroup_version
+                                  when :v2
+                                    read_cgroup_v2_memory_limit
+                                  when :v1
+                                    read_cgroup_v1_memory_limit
+                                  end
+                end
+
+                private
+
+                # Detects which cgroup version is in use
+                # @return [Symbol, nil] :v2, :v1, or nil if not in a cgroup environment
+                def cgroup_version
+                  return @cgroup_version if instance_variable_defined?(:@cgroup_version)
+
+                  @cgroup_version = if File.exist?(CGROUP_V2_CONTROLLERS)
+                                      :v2
+                                    elsif File.exist?(CGROUP_V1_MEMORY_LIMIT)
+                                      :v1
+                                    end
+                end
+
+                # Reads memory limit from cgroup v2
+                # @return [Integer, nil] memory limit in kilobytes, or nil
+                def read_cgroup_v2_memory_limit
+                  return nil unless File.exist?(CGROUP_V2_MEMORY_LIMIT)
+
+                  limit = File.read(CGROUP_V2_MEMORY_LIMIT).strip
+
+                  # "max" means no limit
+                  return nil if limit == CGROUP_V2_MAX
+
+                  # Convert from bytes to kilobytes
+                  limit.to_i / 1024
+                rescue StandardError
+                  nil
+                end
+
+                # Reads memory limit from cgroup v1
+                # @return [Integer, nil] memory limit in kilobytes, or nil
+                def read_cgroup_v1_memory_limit
+                  return nil unless File.exist?(CGROUP_V1_MEMORY_LIMIT)
+
+                  limit = File.read(CGROUP_V1_MEMORY_LIMIT).strip.to_i
+
+                  # Very large values (close to max int64) mean no limit
+                  # Using a threshold of 2^60 as a reasonable "unlimited" indicator
+                  return nil if limit > (2**60)
+
+                  # Convert from bytes to kilobytes
+                  limit / 1024
+                rescue StandardError
+                  nil
+                end
+              end
+
+              # @return [Integer] total amount of memory in kilobytes
+              # In containerized environments, returns the container's memory limit from cgroups.
+              # Falls back to host memory if no limit is set.
+              # @note Memoized at instance level to avoid repeated class method calls
+              def memory_size
+                @memory_size ||= self.class.memory_limit || super
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/consumers/sampler/metrics/jobs.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Consumers
+        class Sampler < Tracking::Sampler
+          module Metrics
+            # Collects job queue statistics and worker utilization metrics
+            class Jobs < Base
+              include ::Karafka::Core::Helpers::Time
+
+              # @param windows [Helpers::Ttls::Windows] time windows for aggregating metrics
+              # @param started_at [Float] process start time
+              # @param workers [Integer] number of worker threads
+              def initialize(windows, started_at, workers)
+                super()
+                @windows = windows
+                @started_at = started_at
+                @workers = workers
+              end
+
+              # @return [Numeric] % utilization of all the threads. 100% means all the threads are
+              #   utilized all the time within the given time window. 0% means, nothing is happening
+              #   most if not all the time.
+              def utilization
+                totals = windows.m1[:processed_total_time]
+
+                return 0 if totals.empty?
+
+                timefactor = [float_now - started_at, 60].min
+
+                # We divide by 1_000 to convert from milliseconds
+                # We multiply by 100 to have it in % scale
+                (totals.sum / 1_000 / workers / timefactor * 100).round(2)
+              end
+
+              # @return [Hash] job queue statistics
+              def jobs_queue_statistics
+                # We return empty stats in case jobs queue is not yet initialized
+                base = Karafka::Server.jobs_queue&.statistics || { busy: 0, enqueued: 0 }
+                stats = base.slice(:busy, :enqueued, :waiting)
+                stats[:waiting] ||= 0
+                # busy - represents number of jobs that are being executed currently
+                # enqueued - jobs that are in the queue but not being picked up yet
+                # waiting - jobs that are not scheduled on the queue but will be
+                # be enqueued in case of advanced schedulers
+                stats
+              end
+
+              private
+
+              attr_reader :windows, :started_at, :workers
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/consumers/sampler/metrics/network.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Consumers
+        class Sampler < Tracking::Sampler
+          module Metrics
+            # Collects network throughput metrics (bytes received/sent per second)
+            class Network < Base
+              # @param windows [Helpers::Ttls::Windows] time windows for aggregating metrics
+              def initialize(windows)
+                super()
+                @windows = windows
+              end
+
+              # @return [Integer] number of bytes received per second out of a one minute time window
+              #   by all the consumers
+              # @note We use one minute window to compensate for cases where metrics would be reported
+              #   or recorded faster or slower. This normalizes data
+              def bytes_received
+                windows
+                  .m1
+                  .stats_from { |k, _v| k.end_with?('rxbytes') }
+                  .rps
+                  .round
+              end
+
+              # @return [Integer] number of bytes sent per second out of a one minute time window by
+              #   all the consumers
+              def bytes_sent
+                windows
+                  .m1
+                  .stats_from { |k, _v| k.end_with?('txbytes') }
+                  .rps
+                  .round
+              end
+
+              private
+
+              attr_reader :windows
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/consumers/sampler/metrics/os.rb

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Consumers
+        class Sampler < Tracking::Sampler
+          module Metrics
+            # Collects OS-level metrics from /proc filesystem and system commands
+            # Used when running directly on a host OS (not in containers)
+            class Os < Base
+              # @param shell [MemoizedShell] shell executor for running system commands
+              def initialize(shell)
+                super()
+                @shell = shell
+              end
+
+              # @return [Integer] memory used by this process in kilobytes (RSS - Resident Set Size)
+              # This is the amount of physical memory currently used by the Karafka process.
+              # On Linux: reads VmRSS from /proc/{pid}/status
+              # On macOS: uses ps command to get RSS for current process
+              # @note This represents ONLY the current Karafka process memory usage
+              def memory_usage
+                pid = ::Process.pid
+
+                case RUBY_PLATFORM
+                # Reading this that way is cheaper than running a shell command
+                when /linux/
+                  File.readlines("/proc/#{pid}/status").each do |line|
+                    next unless line.start_with?('VmRSS:')
+
+                    break line.split[1].to_i
+                  end
+                when /darwin|bsd/
+                  shell
+                    .call("ps -o pid,rss -p #{pid}")
+                    .lines
+                    .last
+                    .split
+                    .last
+                    .to_i
+                else
+                  0
+                end
+              end
+
+              # @param memory_threads_ps [Array, false] parsed ps/proc output for all processes
+              # @return [Integer] total memory used by all processes in the system (or container)
+              # This represents system-wide (or container-wide) memory usage by summing RSS
+              # across all processes.
+              # On bare metal: sums memory for all processes on the host
+              # In containers: sums memory for all processes within the container (due to PID namespace)
+              # @note This is DIFFERENT from memory_usage which only shows current process memory
+              # @note Used in Web UI to show "OS memory used" metric
+              def memory_total_usage(memory_threads_ps)
+                return 0 unless memory_threads_ps
+
+                memory_threads_ps.sum(&:first)
+              end
+
+              # @return [Integer] total amount of available memory in kilobytes
+              # This is the total physical memory available to the system/container.
+              # On Linux: reads MemTotal from /proc/meminfo
+              # On macOS: uses sysctl hw.memsize
+              # In containers: Container class overrides this to return cgroup memory limit
+              # @note This is a STATIC value (system RAM capacity), memoized for performance
+              # @note Used in Web UI to show "OS memory available" metric
+              def memory_size
+                return @memory_size if instance_variable_defined?(:@memory_size)
+
+                @memory_size = case RUBY_PLATFORM
+                               when /linux/
+                                 mem_info = File.read('/proc/meminfo')
+                                 mem_total_line = mem_info.match(/MemTotal:\s*(?<total>\d+)/)
+                                 mem_total_line['total'].to_i
+                               when /darwin|bsd/
+                                 shell
+                               .call('sysctl -a')
+                               .split("\n")
+                               .find { |line| line.start_with?('hw.memsize:') }
+                               .to_s
+                               .split
+                               .last
+                               .to_i
+                               else
+                                 0
+                               end
+              end
+
+              # @return [Array<Float>] load averages for last 1, 5 and 15 minutes
+              def cpu_usage
+                case RUBY_PLATFORM
+                when /linux/
+                  File
+                    .read('/proc/loadavg')
+                    .split
+                    .first(3)
+                    .map(&:to_f)
+                when /darwin|bsd/
+                  shell
+                    .call('w | head -1')
+                    .strip
+                    .split
+                    .map(&:to_f)
+                    .last(3)
+                else
+                  [-1, -1, -1]
+                end
+              end
+
+              # @return [Integer] CPU count
+              def cpus
+                @cpus ||= Etc.nprocessors
+              end
+
+              # @param memory_threads_ps [Array, false] parsed ps output
+              # @return [Integer] number of process threads
+              # @note This returns total number of threads from the OS perspective including
+              #   native extensions threads, etc.
+              def threads(memory_threads_ps)
+                return 0 unless memory_threads_ps
+
+                memory_threads_ps.find { |row| row.last == ::Process.pid }[1]
+              end
+
+              # Loads process information for all running processes
+              # @return [Array<Array<Integer, Integer, Integer>>, false] array of [rss_kb, threads, pid]
+              #   for each process, or false if unavailable
+              #
+              # This method reads information about ALL processes on the system (or in the container).
+              # The data is used by multiple metrics:
+              # - memory_total_usage: sums RSS across all processes
+              # - threads: extracts thread count for current process
+              #
+              # Format of each array element: [memory_in_kb, thread_count, process_id]
+              # - memory_in_kb: RSS (Resident Set Size) in kilobytes
+              # - thread_count: Number of threads (only populated for current process, 0 for others)
+              # - process_id: Process ID
+              #
+              # Platform behavior:
+              # - Linux: Reads /proc/[0-9]*/statm for ALL processes on host/container
+              # - macOS: Uses `ps -A` to get all processes
+              # - Containers: Due to PID namespaces, only sees processes within the container
+              #
+              # @note Sampler calls this once per sample cycle (every ~5 seconds) and caches the result
+              #   in @memory_threads_ps to ensure consistent data within a single sample snapshot
+              # @note The cache is refreshed on EVERY sample cycle, so data stays current
+              # @note On Linux, thread count is only extracted for the current process to optimize performance
+              def memory_threads_ps
+                case RUBY_PLATFORM
+                when /linux/
+                  page_size = Helpers::Sysconf.page_size
+                  current_pid = ::Process.pid
+
+                  # Read all processes from /proc
+                  Dir.glob('/proc/[0-9]*/statm').filter_map do |statm_file|
+                    pid = statm_file.match(%r{/proc/(\d+)/statm})[1].to_i
+                    status_file = "/proc/#{pid}/status"
+
+                    # Extract RSS from /proc/<pid>/statm (second field)
+                    rss_pages = begin
+                      File.read(statm_file).split[1].to_i
+                    rescue StandardError
+                      next # Process may have exited
+                    end
+
+                    # Extract thread count from /proc/<pid>/status (only for current process)
+                    thcount = if pid == current_pid
+                                begin
+                                  File.read(status_file)[/^Threads:\s+(\d+)/, 1].to_i
+                                rescue StandardError
+                                  0
+                                end
+                              else
+                                0
+                              end
+
+                    # Convert RSS from pages to kilobytes
+                    rss_kb = (rss_pages * page_size) / 1024
+
+                    [rss_kb, thcount, pid]
+                  end
+                # thcount is not available on macos ps
+                # because of that we inject 0 as threads count similar to how
+                # we do on windows
+                when /darwin|bsd/
+                  shell
+                    .call('ps -A -o rss=,pid=')
+                    .split("\n")
+                    .map { |row| row.strip.split.map(&:to_i) }
+                    .map { |row| [row.first, 0, row.last] }
+                else
+                  false
+                end
+              end
+
+              private
+
+              attr_reader :shell
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/consumers/sampler/metrics/server.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Consumers
+        class Sampler < Tracking::Sampler
+          module Metrics
+            # Collects Karafka server state metrics (listeners, workers, status)
+            class Server < Base
+              # @return [Hash] number of active and standby listeners
+              def listeners
+                if Karafka::Server.listeners
+                  active = Karafka::Server.listeners.count(&:active?)
+                  total = Karafka::Server.listeners.count.to_i
+
+                  { active: active, standby: total - active }
+                else
+                  { active: 0, standby: 0 }
+                end
+              end
+
+              # @return [Integer] number of threads that process work
+              def workers
+                Karafka::App.config.concurrency
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end