karafka-web 0.11.5 → 0.11.6

data/lib/karafka/web/processing/consumers/aggregators/metrics.rb

@@ -11,7 +11,7 @@ module Karafka
           class Metrics < Base
             # Current schema version
             # This is used for detecting incompatible changes and writing migrations
-            SCHEMA_VERSION = '1.3.0'
+            SCHEMA_VERSION = "1.3.0"
 
             def initialize
               super
@@ -43,8 +43,8 @@ module Karafka
             # @return [Hash] Statistics hash
             #
             # @note We materialize the consumers groups time series only here and not in real time,
-            #   because we materialize it based on the tracked active collective state. Materializing
-            #   on each update that would not be dispatched would be pointless.
+            #   because we materialize it based on the tracked active collective state.
+            #   Materializing on each update that would not be dispatched would be pointless.
             def to_h
               metrics[:schema_version] = SCHEMA_VERSION
               metrics[:dispatched_at] = float_now
@@ -71,7 +71,7 @@ module Karafka
               max_ttl = @aggregated_from - (::Karafka::Web.config.ttl / 1_000)
 
               @active_reports.delete_if do |_id, report|
-                report[:dispatched_at] < max_ttl || report[:process][:status] == 'stopped'
+                report[:dispatched_at] < max_ttl || report[:process][:status] == "stopped"
               end
             end
 
@@ -93,15 +93,15 @@ module Karafka
 
               iterate_partitions_data do |group_name, topic_name, partitions_data|
                 lags_hybrid = partitions_data
-                              .map do |p_details|
-                                lag_stored = p_details.fetch(:lag_stored, -1)
-                                lag_stored.negative? ? p_details.fetch(:lag, -1) : lag_stored
-                              end
-                              .reject(&:negative?)
+                  .map do |p_details|
+                    lag_stored = p_details.fetch(:lag_stored, -1)
+                    lag_stored.negative? ? p_details.fetch(:lag, -1) : lag_stored
+                end
+                  .reject(&:negative?)
 
                 offsets_hi = partitions_data
-                             .map { |p_details| p_details.fetch(:hi_offset, -1) }
-                             .reject(&:negative?)
+                  .map { |p_details| p_details.fetch(:hi_offset, -1) }
+                  .reject(&:negative?)
 
                 # Last stable offsets freeze durations - we pick the max freeze to indicate
                 # the longest open transaction that potentially may be hanging

data/lib/karafka/web/processing/consumers/aggregators/state.rb

@@ -20,7 +20,7 @@ module Karafka
             # Current schema version
             # This can be used in the future for detecting incompatible changes and writing
             # migrations
-            SCHEMA_VERSION = '1.4.0'
+            SCHEMA_VERSION = "1.4.0"
 
             # @param schema_manager [Karafka::Web::Processing::Consumers::SchemaManager] schema
             #   manager that tracks the compatibility of schemas.
@@ -118,7 +118,8 @@ module Karafka
               end
             end
 
-            # Refreshes the counters that are computed based on incoming reports and not a total sum.
+            # Refreshes the counters that are computed based on incoming reports and not a
+            # total sum.
             # For this we use active reports we have in memory. It may not be accurate for the first
             # few seconds but it is much more optimal from performance perspective than computing
             # this fetching all data from Kafka for each view.
@@ -138,7 +139,7 @@ module Karafka
 
               @active_reports
                 .values
-                .reject { |report| report[:process][:status] == 'stopped' }
+                .reject { |report| report[:process][:status] == "stopped" }
                 .each do |report|
                   report_stats = report[:stats]
                   report_process = report[:process]

data/lib/karafka/web/processing/consumers/reports_migrator.rb

@@ -39,8 +39,8 @@ module Karafka
           # Only computed when first needed to avoid memory overhead if no old reports exist
           def migrations
             @migrations ||= Management::Migrations::Base
-                            .sorted_descendants
-                            .select { |migration_class| migration_class.type == :consumers_reports }
+              .sorted_descendants
+              .select { |migration_class| migration_class.type == :consumers_reports }
           end
         end
       end

data/lib/karafka/web/processing/consumers/schema_manager.rb

@@ -54,7 +54,7 @@ module Karafka
 
           # @return [String] state that we can use in the materialized state for the UI reporting
           def to_s
-            @valid ? 'compatible' : 'incompatible'
+            @valid ? "compatible" : "incompatible"
           end
         end
       end

data/lib/karafka/web/processing/publisher.rb

@@ -41,14 +41,14 @@ module Karafka
                 # This will ensure that the consumer states are compacted
                 key: Karafka::Web.config.topics.consumers.states.name,
                 partition: 0,
-                headers: { 'zlib' => 'true' }
+                headers: { "zlib" => "true" }
               },
               {
                 topic: Karafka::Web.config.topics.consumers.metrics.name,
                 payload: Zlib::Deflate.deflate(consumers_metrics.to_json),
                 key: Karafka::Web.config.topics.consumers.metrics.name,
                 partition: 0,
-                headers: { 'zlib' => 'true' }
+                headers: { "zlib" => "true" }
               }
             ]
           end

data/lib/karafka/web/producer.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    # A lazy-evaluated producer wrapper that creates a low-intensity variant of the default
+    # Karafka producer when possible.
+    #
+    # Web UI reporting is not mission-critical and serves primarily analytical purposes.
+    # Users typically want stronger delivery warranties for their business producers.
+    # For web UI reporting, we can use lower acknowledgment levels to reduce overhead.
+    #
+    # This wrapper:
+    # - Returns the default producer unchanged if it's idempotent or transactional
+    #   (since acks cannot be altered for these producer types)
+    # - Creates a variant with `acks: 1` for non-idempotent, non-transactional producers
+    #   to reduce latency while maintaining basic delivery confirmation
+    #
+    # @note This uses SimpleDelegator to transparently proxy all producer methods
+    # @note The variant is created lazily on first access to ensure the default producer
+    #   is fully initialized
+    class Producer < SimpleDelegator
+      def initialize
+        @initialized = false
+        # Initialize with nil - will be set on first access
+        super(nil)
+      end
+
+      # @return [WaterDrop::Producer, WaterDrop::Producer::Variant] the underlying producer
+      #   or its low-ack variant
+      def __getobj__
+        unless @initialized
+          @delegate_sd_obj = build_producer
+          @initialized = true
+        end
+
+        @delegate_sd_obj
+      end
+
+      private
+
+      # Builds the appropriate producer based on the default producer's configuration
+      #
+      # @return [WaterDrop::Producer, WaterDrop::Producer::Variant] either the default producer
+      #   (if idempotent/transactional) or a low-ack variant
+      def build_producer
+        default = ::Karafka.producer
+
+        # Idempotent producers require acks: all - cannot create variants with different acks
+        return default if default.idempotent?
+        # Transactional producers also require acks: all
+        return default if default.transactional?
+
+        # For non-idempotent, non-transactional producers, create a variant with lower acks
+        # acks: 0 means fire-and-forget - no acknowledgment required
+        # This is acceptable for non-critical analytics/monitoring data where occasional
+        # message loss is not a concern
+        default.variant(topic_config: { acks: 0 })
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/consumers/contracts/report.rb

@@ -16,11 +16,11 @@ module Karafka
             required(:dispatched_at) { |val| val.is_a?(Numeric) && val.positive? }
             # We have consumers and producer reports and need to ensure that each is handled
             # in an expected fashion
-            required(:type) { |val| val == 'consumer' }
+            required(:type) { |val| val == "consumer" }
 
             nested(:process) do
               required(:started_at) { |val| val.is_a?(Numeric) && val.positive? }
-              required(:id) { |val| val.is_a?(String) && val.count(':') >= 2 }
+              required(:id) { |val| val.is_a?(String) && val.count(":") >= 2 }
               required(:cpus) { |val| val.is_a?(Integer) && val >= 1 }
               required(:memory_usage) { |val| val.is_a?(Integer) && val >= 0 }
               required(:memory_total_usage) { |val| val.is_a?(Integer) && val >= 0 }

data/lib/karafka/web/tracking/consumers/contracts/subscription_group.rb

@@ -12,6 +12,7 @@ module Karafka
 
             required(:id) { |val| val.is_a?(String) && !val.empty? }
             required(:topics) { |val| val.is_a?(Hash) }
+            required(:instance_id) { |val| val == false || val.is_a?(String) }
 
             nested(:state) do
               required(:state) { |val| val.is_a?(String) && !val.empty? }

data/lib/karafka/web/tracking/consumers/listeners/booting.rb

@@ -15,7 +15,7 @@ module Karafka
             # @param _event [Karafka::Core::Monitoring::Event]
             def on_app_running(_event)
               ::Karafka::Web.config.tracking.scheduler.async_call(
-                'karafka.web.tracking.scheduler'
+                "karafka.web.tracking.scheduler"
               )
             end
 

data/lib/karafka/web/tracking/consumers/listeners/connections.rb

@@ -10,11 +10,16 @@ module Karafka
             # Initializes the subscription group with defaults so it is always available
             # @param event [Karafka::Core::Monitoring::Event]
             def on_connection_listener_before_fetch_loop(event)
-              sg_id = event[:subscription_group].id
+              subscription_group = event[:subscription_group]
+              sg_id = subscription_group.id
+              # Use false to explicitly indicate static membership is not configured
+              # vs nil which could be ambiguous
+              instance_id = subscription_group.kafka[:"group.instance.id"] || false
 
               track do |sampler|
                 # This will initialize the hash upon first request
-                sampler.subscription_groups[sg_id]
+                sg = sampler.subscription_groups[sg_id]
+                sg[:instance_id] = instance_id
               end
             end
 

data/lib/karafka/web/tracking/consumers/listeners/errors.rb

@@ -10,7 +10,7 @@ module Karafka
             include Tracking::Helpers::ErrorInfo
 
             # Schema used by consumers error reporting
-            SCHEMA_VERSION = '1.2.0'
+            SCHEMA_VERSION = "1.2.0"
 
             private_constant :SCHEMA_VERSION
 
@@ -23,15 +23,15 @@ module Karafka
               # Collect extra info if it was a consumer related error.
               # Those come from user code
               details = case caller_ref
-                        when Karafka::BaseConsumer
-                          extract_consumer_info(caller_ref)
-                        when Karafka::Connection::Client
-                          extract_client_info(caller_ref)
-                        when Karafka::Connection::Listener
-                          extract_listener_info(caller_ref)
-                        else
-                          {}
-                        end
+              when Karafka::BaseConsumer
+                extract_consumer_info(caller_ref)
+              when Karafka::Connection::Client
+                extract_client_info(caller_ref)
+              when Karafka::Connection::Listener
+                extract_listener_info(caller_ref)
+              else
+                {}
+              end
 
               error_class, error_message, backtrace = extract_error_info(event[:error])
 

data/lib/karafka/web/tracking/consumers/listeners/pausing.rb

@@ -38,7 +38,7 @@ module Karafka
               partition = event[:partition]
               subscription_group_id = event[:subscription_group].id
 
-              [subscription_group_id, topic, partition].join('-')
+              [subscription_group_id, topic, partition].join("-")
             end
           end
         end

data/lib/karafka/web/tracking/consumers/listeners/processing.rb

@@ -68,8 +68,8 @@ module Karafka
             def on_consumer_consume(event)
               consumer = event.payload[:caller]
               messages_count = consumer.messages.size
-              jid = job_id(consumer, 'consume')
-              job_details = job_details(consumer, 'consume')
+              jid = job_id(consumer, "consume")
+              job_details = job_details(consumer, "consume")
 
               track do |sampler|
                 # We count batches and messages prior to the execution, so they are tracked even
@@ -87,7 +87,7 @@ module Karafka
             # @param event [Karafka::Core::Monitoring::Event]
             def on_consumer_consumed(event)
               consumer = event.payload[:caller]
-              jid = job_id(consumer, 'consume')
+              jid = job_id(consumer, "consume")
 
               track do |sampler|
                 sampler.jobs.delete(jid)
@@ -100,23 +100,23 @@ module Karafka
             def on_error_occurred(event)
               track do |sampler|
                 type = case event[:type]
-                       when 'consumer.consume.error'
-                         'consume'
-                       when 'consumer.revoked.error'
-                         'revoked'
-                       when 'consumer.shutdown.error'
-                         'shutdown'
-                       when 'consumer.tick.error'
-                         'tick'
-                       when 'consumer.eofed.error'
-                         'eofed'
-                       # This is not a user facing execution flow, but internal system one
-                       # that is why it will not be reported as a separate job for the UI
-                       when 'consumer.idle.error'
-                         false
-                       else
-                         false
-                       end
+                when "consumer.consume.error"
+                  "consume"
+                when "consumer.revoked.error"
+                  "revoked"
+                when "consumer.shutdown.error"
+                  "shutdown"
+                when "consumer.tick.error"
+                  "tick"
+                when "consumer.eofed.error"
+                  "eofed"
+                # This is not a user facing execution flow, but internal system one
+                # that is why it will not be reported as a separate job for the UI
+                when "consumer.idle.error"
+                  false
+                else
+                  false
+                end
 
                 # job reference only exists for consumer work related operations.
                 # Only for them we need to deregister the job reference.
@@ -132,10 +132,10 @@ module Karafka
             # Consume has a bit different reporting flow than other jobs because it bumps certain
             # counters that other jobs do not. This is why it is defined above separately
             [
-              [:revoke, :revoked, 'revoked'],
-              [:shutting_down, :shutdown, 'shutdown'],
-              [:tick, :ticked, 'tick'],
-              [:eof, :eofed, 'eofed']
+              [:revoke, :revoked, "revoked"],
+              [:shutting_down, :shutdown, "shutdown"],
+              [:tick, :ticked, "tick"],
+              [:eof, :eofed, "eofed"]
             ].each do |pre, post, action|
               # Dynamically creates methods like:
               #   def on_consumer_revoke(event)
@@ -227,7 +227,7 @@ module Karafka
                 consumer_group: consumer.topic.consumer_group.id,
                 type: type,
                 tags: consumer.tags,
-                status: 'running'
+                status: "running"
               }
             end
           end

data/lib/karafka/web/tracking/consumers/listeners/statistics.rb

@@ -12,8 +12,8 @@ module Karafka
             # @param event [Karafka::Core::Monitoring::Event]
             def on_statistics_emitted(event)
               statistics = event[:statistics]
-              topics = statistics.fetch('topics')
-              cgrp = statistics.fetch('cgrp')
+              topics = statistics.fetch("topics")
+              cgrp = statistics.fetch("cgrp")
               cg_id = event[:consumer_group_id]
               sg_id = event[:subscription_group_id]
               sg_details = extract_sg_details(sg_id, cgrp)
@@ -25,7 +25,7 @@ module Karafka
               # in track as we merge data from multiple subscription groups
               track do |sampler|
                 topics.each do |topic_name, topic_values|
-                  partitions = topic_values.fetch('partitions')
+                  partitions = topic_values.fetch("partitions")
 
                   partitions.each do |pt_name, pt_stats|
                     pt_id = pt_name.to_i
@@ -64,18 +64,18 @@ module Karafka
             #
             # @param statistics [Hash] statistics hash
             def track_transfers(statistics)
-              brokers = statistics.fetch('brokers', {})
+              brokers = statistics.fetch("brokers", {})
 
               return if brokers.empty?
 
               track do |sampler|
-                client_name = statistics.fetch('name')
+                client_name = statistics.fetch("name")
 
                 brokers.each do |broker_name, values|
                   scope_name = "#{client_name}-#{broker_name}"
 
-                  sampler.windows.m1["#{scope_name}-rxbytes"] << values.fetch('rxbytes', 0)
-                  sampler.windows.m1["#{scope_name}-txbytes"] << values.fetch('txbytes', 0)
+                  sampler.windows.m1["#{scope_name}-rxbytes"] << values.fetch("rxbytes", 0)
+                  sampler.windows.m1["#{scope_name}-txbytes"] << values.fetch("txbytes", 0)
                 end
               end
             end
@@ -88,12 +88,12 @@ module Karafka
               {
                 id: sg_id,
                 state: sg_stats.slice(
-                  'state',
-                  'join_state',
-                  'stateage',
-                  'rebalance_age',
-                  'rebalance_cnt',
-                  'rebalance_reason'
+                  "state",
+                  "join_state",
+                  "stateage",
+                  "rebalance_age",
+                  "rebalance_cnt",
+                  "rebalance_reason"
                 ).transform_keys(&:to_sym),
                 topics: {}
               }
@@ -108,11 +108,11 @@ module Karafka
 
               # Collect information only about what we are subscribed to and what we fetch or
               # work in any way. Stopped means, we stopped working with it
-              return false if pt_stats['fetch_state'] == 'stopped'
+              return false if pt_stats["fetch_state"] == "stopped"
 
               # Return if we no longer fetch this partition in a particular process. None means
               # that we no longer have this subscription assigned and we do not fetch
-              return false if pt_stats['fetch_state'] == 'none'
+              return false if pt_stats["fetch_state"] == "none"
 
               true
             end
@@ -123,29 +123,29 @@ module Karafka
             # @return [Hash] extracted partition metrics
             def extract_partition_metrics(pt_stats)
               metrics = pt_stats.slice(
-                'consumer_lag',
-                'consumer_lag_d',
-                'consumer_lag_stored',
-                'consumer_lag_stored_d',
-                'committed_offset',
+                "consumer_lag",
+                "consumer_lag_d",
+                "consumer_lag_stored",
+                "consumer_lag_stored_d",
+                "committed_offset",
                 # Can be useful to track the frequency of flushes when there is progress
-                'committed_offset_fd',
-                'stored_offset',
+                "committed_offset_fd",
+                "stored_offset",
                 # Can be useful to track the frequency of flushes when there is progress
-                'stored_offset_fd',
-                'fetch_state',
-                'hi_offset',
-                'hi_offset_fd',
-                'lo_offset',
-                'eof_offset',
-                'ls_offset',
+                "stored_offset_fd",
+                "fetch_state",
+                "hi_offset",
+                "hi_offset_fd",
+                "lo_offset",
+                "eof_offset",
+                "ls_offset",
                 # Two below can be useful for detection of hanging transactions
-                'ls_offset_d',
-                'ls_offset_fd'
+                "ls_offset_d",
+                "ls_offset_fd"
               )
 
               # Rename as we do not need `consumer_` prefix
-              metrics.transform_keys! { |key| key.gsub('consumer_', '') }
+              metrics.transform_keys! { |key| key.gsub("consumer_", "") }
               metrics.transform_keys!(&:to_sym)
 
               metrics
@@ -156,16 +156,16 @@ module Karafka
             # @param pt_id [Integer] partition id
             # @return [String] poll state / is partition paused or not
             def poll_details(sg_id, topic_name, pt_id)
-              pause_id = [sg_id, topic_name, pt_id].join('-')
+              pause_id = [sg_id, topic_name, pt_id].join("-")
 
-              details = { poll_state: 'active', poll_state_ch: 0 }
+              details = { poll_state: "active", poll_state_ch: 0 }
 
               pause_details = sampler.pauses[pause_id]
 
               return details unless pause_details
 
               {
-                poll_state: 'paused',
+                poll_state: "paused",
                 poll_state_ch: [(pause_details.fetch(:paused_till) - monotonic_now).round, 0].max
               }
             end

data/lib/karafka/web/tracking/consumers/reporter.rb

@@ -71,7 +71,7 @@ module Karafka
                   payload: Zlib::Deflate.deflate(report.to_json),
                   key: process_id,
                   partition: 0,
-                  headers: { 'zlib' => 'true' }
+                  headers: { "zlib" => "true" }
                 }
               ]
 
@@ -84,7 +84,7 @@ module Karafka
                   payload: Zlib::Deflate.deflate(error.to_json),
                   # Always dispatch errors from the same process to the same partition
                   key: process_id,
-                  headers: { 'zlib' => 'true' }
+                  headers: { "zlib" => "true" }
                 }
               end
 

data/lib/karafka/web/tracking/consumers/sampler/enrichers/consumer_groups.rb

@@ -50,6 +50,7 @@ module Karafka
 
                 polled_at = sg_tracking.fetch(:polled_at)
                 sg_details[:state][:poll_age] = (monotonic_now - polled_at).round(2)
+                sg_details[:instance_id] = sg_tracking[:instance_id]
 
                 sg_details[:topics].each do |topic_name, topic_details|
                   topic_details[:partitions].each do |partition_id, partition_details|

data/lib/karafka/web/tracking/consumers/sampler/metrics/base.rb

@@ -8,7 +8,8 @@ module Karafka
           # Namespace for metrics collectors that gather various system and process statistics
           module Metrics
             # Base class for metrics collectors
-            # This is an abstract base class that can be extended to create custom metrics collectors
+            # This is an abstract base class that can be extended to create custom metrics
+            # collectors
             class Base
               # Placeholder for future common functionality
             end

data/lib/karafka/web/tracking/consumers/sampler/metrics/container.rb

@@ -11,16 +11,16 @@ module Karafka
             # Supports both cgroups v1 and v2
             class Container < Os
               # Maximum value that represents "no limit" in cgroup v2
-              CGROUP_V2_MAX = 'max'
+              CGROUP_V2_MAX = "max"
 
               # Paths for cgroup detection and reading
-              CGROUP_V2_CONTROLLERS = '/sys/fs/cgroup/cgroup.controllers'
+              CGROUP_V2_CONTROLLERS = "/sys/fs/cgroup/cgroup.controllers"
 
               # Memory paths
               # Path to cgroup v1 memory limit file
-              CGROUP_V1_MEMORY_LIMIT = '/sys/fs/cgroup/memory/memory.limit_in_bytes'
+              CGROUP_V1_MEMORY_LIMIT = "/sys/fs/cgroup/memory/memory.limit_in_bytes"
               # Path to cgroup v2 memory limit file
-              CGROUP_V2_MEMORY_LIMIT = '/sys/fs/cgroup/memory.max'
+              CGROUP_V2_MEMORY_LIMIT = "/sys/fs/cgroup/memory.max"
 
               private_constant(
                 :CGROUP_V2_MAX,
@@ -42,11 +42,11 @@ module Karafka
                   return @memory_limit if instance_variable_defined?(:@memory_limit)
 
                   @memory_limit = case cgroup_version
-                                  when :v2
-                                    read_cgroup_v2_memory_limit
-                                  when :v1
-                                    read_cgroup_v1_memory_limit
-                                  end
+                  when :v2
+                    read_cgroup_v2_memory_limit
+                  when :v1
+                    read_cgroup_v1_memory_limit
+                  end
                 end
 
                 private
@@ -57,10 +57,10 @@ module Karafka
                   return @cgroup_version if instance_variable_defined?(:@cgroup_version)
 
                   @cgroup_version = if File.exist?(CGROUP_V2_CONTROLLERS)
-                                      :v2
-                                    elsif File.exist?(CGROUP_V1_MEMORY_LIMIT)
-                                      :v1
-                                    end
+                    :v2
+                  elsif File.exist?(CGROUP_V1_MEMORY_LIMIT)
+                    :v1
+                  end
                 end
 
                 # Reads memory limit from cgroup v2
@@ -75,7 +75,7 @@ module Karafka
 
                   # Convert from bytes to kilobytes
                   limit.to_i / 1024
-                rescue StandardError
+                rescue
                   nil
                 end
 
@@ -92,7 +92,7 @@ module Karafka
 
                   # Convert from bytes to kilobytes
                   limit / 1024
-                rescue StandardError
+                rescue
                   nil
                 end
               end

data/lib/karafka/web/tracking/consumers/sampler/metrics/network.rb

@@ -14,14 +14,14 @@ module Karafka
                 @windows = windows
               end
 
-              # @return [Integer] number of bytes received per second out of a one minute time window
-              #   by all the consumers
-              # @note We use one minute window to compensate for cases where metrics would be reported
-              #   or recorded faster or slower. This normalizes data
+              # @return [Integer] number of bytes received per second out of a one minute time
+              #   window by all the consumers
+              # @note We use one minute window to compensate for cases where metrics would be
+              #   reported or recorded faster or slower. This normalizes data
               def bytes_received
                 windows
                   .m1
-                  .stats_from { |k, _v| k.end_with?('rxbytes') }
+                  .stats_from { |k, _v| k.end_with?("rxbytes") }
                   .rps
                   .round
               end
@@ -31,7 +31,7 @@ module Karafka
               def bytes_sent
                 windows
                   .m1
-                  .stats_from { |k, _v| k.end_with?('txbytes') }
+                  .stats_from { |k, _v| k.end_with?("txbytes") }
                   .rps
                   .round
               end