karafka-web 0.6.2 → 0.7.0

data/lib/karafka/web/ui/models/metrics/aggregated.rb

@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Models
+        # Namespace for metrics related models
+        module Metrics
+          # Materializes the aggregated data and computes the expected diffs out of the snapshots
+          # We do some pre-processing to make sure, we do not have bigger gaps and to compensate
+          # for reporting drifting
+          class Aggregated < Lib::HashProxy
+            include ::Karafka::Core::Helpers::Time
+
+            # If samples are closer than that, sample will be rejected
+            MIN_ACCEPTED_DRIFT = 4
+
+            # If samples are further away than that, we will inject an artificial sample in-between
+            MAX_ACCEPTED_DRIFT = 7
+
+            # For which keys we should compute the delta in reference to the previous period
+            # Metrics we get from the processes are always absolute, hence we need a reference point
+            # to compute the deltas
+            #
+            # If at least two elements do not exist for given delta range, we keep it empty
+            DELTA_KEYS = %i[
+              batches
+              messages
+              errors
+              retries
+              dead
+            ].freeze
+
+            private_constant :MIN_ACCEPTED_DRIFT, :MAX_ACCEPTED_DRIFT, :DELTA_KEYS
+
+            # Builds the Web-UI historicals representation that includes deltas
+            #
+            # @param aggregated [Hash] aggregated historical metrics
+            def initialize(aggregated)
+              aggregated
+                .tap { |historicals| reject_drifters(historicals) }
+                .tap { |historicals| fill_gaps(historicals) }
+                .then { |historicals| enrich_with_deltas(historicals) }
+                .tap { |historicals| enrich_with_batch_size(historicals) }
+                .tap { |historicals| enrich_with_process_rss(historicals) }
+                .then { |enriched| super(enriched) }
+            end
+
+            # @return [Boolean] do we have enough data to draw any basic charts
+            def sufficient?
+              seconds.size > 2
+            end
+
+            private
+
+            # Since our reporting is not ms precise, there are cases where sampling can drift.
+            # If drifting gets us close to one side, for delta metrics it would create sudden
+            # artificial drops in metrics that would not match the reality. We reject drifters like
+            # this as we can compensate this later.
+            #
+            # This problems only affects our near real-time metrics with seconds precision
+            #
+            # @param historicals [Hash] all historicals for all the ranges
+            def reject_drifters(historicals)
+              initial = nil
+
+              historicals.fetch(:seconds).delete_if do |sample|
+                unless initial
+                  initial = sample.first
+
+                  next
+                end
+
+                # Reject values that are closer than minimum
+                too_close = sample.first - initial < MIN_ACCEPTED_DRIFT
+
+                initial = sample.first
+
+                too_close
+              end
+            end
+
+            # In case of a positive drift, we may have gaps bigger than few seconds in reporting.
+            # This can create a false sense of spikes that do not reflect the reality. We compensate
+            # this by extrapolating the delta values and using the rest as they are.
+            #
+            # This problems only affects our near real-time metrics with seconds precision
+            #
+            # @param historicals [Hash] all historicals for all the ranges
+            def fill_gaps(historicals)
+              filled = []
+              previous = nil
+
+              historicals.fetch(:seconds).each do |sample|
+                unless previous
+                  filled << sample
+                  previous = sample
+                  next
+                end
+
+                if sample.first - previous.first > MAX_ACCEPTED_DRIFT
+                  base = sample.last.dup
+
+                  DELTA_KEYS.each do |key|
+                    base[key] = previous.last[key] + (sample.last[key] - previous.last[key]) / 2
+                  end
+
+                  filled << [previous.first + (sample.first - previous.first) / 2, base]
+                end
+
+                filled << sample
+                previous = sample
+              end
+
+              historicals[:seconds] = filled
+            end
+
+            # Takes the historical hash, iterates over all the samples and enriches them with the
+            # delta computed values
+            #
+            # @param historicals [Hash] all historicals for all the ranges
+            # @return [Hash] historicals with delta based data
+            def enrich_with_deltas(historicals)
+              results = {}
+
+              historicals.each do |range, time_samples|
+                results[range] = []
+
+                baseline = nil
+
+                time_samples.each do |time_sample|
+                  metrics = time_sample[1]
+
+                  if baseline
+                    deltas = compute_deltas(baseline, metrics)
+                    results[range] << [time_sample[0], metrics.merge(deltas)]
+                  end
+
+                  baseline = metrics
+                end
+              end
+
+              results
+            end
+
+            # Batch size is a match between number of messages and number of batches
+            # It is derived out of the data we have so we compute it on the fly
+            # @param historicals [Hash] all historicals for all the ranges
+            def enrich_with_batch_size(historicals)
+              historicals.each_value do |time_samples|
+                time_samples.each do |time_sample|
+                  metrics = time_sample[1]
+
+                  batches = metrics[:batches]
+
+                  # We check if not zero just in case something would be off there
+                  # We do not want to divide by zero
+                  metrics[:batch_size] = batches.zero? ? 0 : metrics[:messages] / batches
+                end
+              end
+            end
+
+            # Adds an average RSS on a per process basis
+            # @param historicals [Hash] all historicals for all the ranges
+            def enrich_with_process_rss(historicals)
+              historicals.each_value do |time_samples|
+                time_samples.each do |time_sample|
+                  metrics = time_sample[1]
+
+                  rss = metrics[:rss]
+                  processes = metrics[:processes]
+
+                  metrics[:process_rss] = processes.zero? ? 0 : rss / processes
+                end
+              end
+            end
+
+            # Computes deltas for all the relevant keys for which we want to have deltas
+            #
+            # @param previous [Hash]
+            # @param current [Hash]
+            # @return [Hash] delta computed values
+            def compute_deltas(previous, current)
+              DELTA_KEYS.map do |delta_key|
+                [
+                  delta_key,
+                  current.fetch(delta_key) - previous.fetch(delta_key)
+                ]
+              end.to_h
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/models/metrics/charts/aggregated.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Models
+        module Metrics
+          # Namespace for models related to presentation of charts
+          module Charts
+            # Model for formatting aggregated metrics data for charts
+            class Aggregated < Lib::HashProxy
+              # @param aggregated [Hash] all aggregated for all periods
+              # @param period [Symbol] period that we are interested in
+              def initialize(aggregated, period)
+                @data = aggregated.to_h.fetch(period)
+              end
+
+              # @param args [Array<String>] names of aggregated we want to show
+              # @return [String] JSON with data about all the charts we were interested in
+              def with(*args)
+                args
+                  .map { |name| [name.to_sym, public_send(name)] }
+                  .to_h
+                  .to_json
+              end
+
+              # @param method_name [String]
+              # @param include_private [Boolean]
+              def respond_to_missing?(method_name, include_private = false)
+                @data.last.last.key?(method_name.to_sym) || super
+              end
+
+              # Handles delegation to fetch appropriate historical metrics based on their name
+              #
+              # @param method_name [String]
+              # @param arguments [Array] missing method call arguments
+              def method_missing(method_name, *arguments)
+                if @data.last.last.key?(method_name.to_sym)
+                  @data.map { |a| [a.first, a.last[method_name]] }
+                else
+                  super
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/models/metrics/charts/topics.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Models
+        module Metrics
+          module Charts
+            # Model for preparing data about topics states
+            class Topics
+              # @param topics_data [Hash] topics aggregated metrics data
+              # @param period [Symbol] period that we are interested in
+              def initialize(topics_data, period)
+                @data = topics_data.to_h.fetch(period)
+              end
+
+              # @return [String] JSON with lags of each of the topics + total lag of all the topics
+              #   from all the consumer groups.
+              def lags_stored
+                total = Hash.new { |h, v| h[v] = 0 }
+
+                @data.to_h.each_value do |metrics|
+                  metrics.each do |metric|
+                    time = metric.first
+                    lag_stored = metric.last[:lag_stored]
+
+                    if lag_stored
+                      total[time] ||= 0
+                      total[time] += lag_stored
+                    else
+                      next if total.key?(time)
+
+                      total[time] = nil
+                    end
+                  end
+                end
+
+                # Extract the lag stored only from all the data
+                per_topic = @data.to_h.map do |topic, metrics|
+                  extracted = metrics.map { |metric| [metric.first, metric.last[:lag_stored]] }
+
+                  [topic, extracted]
+                end.to_h
+
+                # We name it with a space because someone may have a topic called "total" and we
+                # want to avoid collisions
+                per_topic.merge('total sum' => total.to_a).to_json
+              end
+
+              # @return [String] JSON with producers pace that represents high-watermarks sum for
+              #   each topic
+              def topics_pace
+                topics = {}
+
+                @data.to_h.each do |topic, metrics|
+                  topic_without_cg = topic.split('[').first
+
+                  # If we've already seen this topic data, we can skip
+                  next if topics.include?(topic_without_cg)
+
+                  topics[topic_without_cg] = metrics.map do |current|
+                    [current.first, current.last[:pace]]
+                  end
+                end
+
+                topics.each_value(&:compact!)
+                topics.to_json
+              end
+
+              # @return [String] JSON with per-topic, highest LSO freeze duration. Useful for
+              #   debugging of issues arising from hanging transactions
+              def max_lso_time
+                topics = Hash.new { |h, k| h[k] = Hash.new { |h2, k2| h2[k2] = [] } }
+
+                @data.to_h.each do |topic, metrics|
+                  topic_without_cg = topic.split('[').first
+
+                  metrics.each do |current|
+                    ls_offset = current.last[:ls_offset] || 0
+                    ls_offset_fd = current.last[:ls_offset_fd] || 0
+                    hi_offset = current.last[:hi_offset] || 0
+
+                    # We convert this to seconds from milliseconds due to our Web UI precision
+                    # Reporting is in ms for consistency
+                    normalized_fd = (ls_offset_fd / 1_000).round
+                    # In case ls_offset and hi_offset are the same, it means we're reached eof
+                    # and we just don't have more data. In cases like this, LSO freeze duration
+                    # will grow because LSO will remain unchanged, but it does not mean it is
+                    # frozen. It means there is just no more data in the topic partition
+                    # This means we need to nullify this case, otherwise it would report, that
+                    # lso is hanging.
+                    normalized_fd = 0 if ls_offset == hi_offset
+
+                    topics[topic_without_cg][current.first] << normalized_fd
+                  end
+                end
+
+                topics.each_value(&:compact!)
+                topics.each_value { |metrics| metrics.transform_values!(&:max) }
+                topics.transform_values! { |values| values.to_a.sort_by!(&:first) }
+                topics.to_json
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/models/metrics/topics.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Models
+        module Metrics
+          # Representation of topics historical metrics based on the aggregated metrics data
+          # We do some pre-processing to align and normalize all the data
+          class Topics < Lib::HashProxy
+            # @param consumers_groups [Hash] historical metrics for consumers groups
+            def initialize(consumers_groups)
+              aggregate_topics_data(consumers_groups)
+                .tap { |topics_metrics| nulify_gaps(topics_metrics) }
+                .then { |topics_metrics| super(topics_metrics) }
+            end
+
+            private
+
+            # Extracts and aggregates data on a per-topic basis in a hash. Because in theory same
+            # topic can be consumed by multiple consumer groups, we include consumer group in the
+            # hash keys.
+            #
+            # @param consumers_groups [Hash] consumers groups initial hash with metrics
+            # @return [Hash] remapped hash with range including extracted topics details
+            def aggregate_topics_data(consumers_groups)
+              extracted = Hash.new { |h, k| h[k] = [] }
+
+              consumers_groups.each do |range, samples|
+                range_extracted = {}
+
+                samples.each do |sample|
+                  time = sample.first
+                  groups = sample.last
+
+                  groups.each do |cg_name, topics|
+                    topics.each do |topic_name, topic_data|
+                      range_extracted["#{topic_name}[#{cg_name}]"] ||= []
+                      range_extracted["#{topic_name}[#{cg_name}]"] << [time, topic_data]
+                    end
+                  end
+                end
+
+                # Always align the order of topics in hash based on their name so it is
+                # independent from the reported order
+                extracted[range] = range_extracted.keys.sort.map do |key|
+                  [key, range_extracted[key]]
+                end.to_h
+              end
+
+              extracted
+            end
+
+            # Nullifies gaps within data with metrics with nil values. This is needed for us to be
+            # able to provide consistent charts even with gaps in reporting.
+            #
+            # @param topics_metrics [Hash] flattened topics data
+            # @note This modifies the original data in place
+            # @note We nullify both gaps in metrics as well as gaps in times (no values for time)
+            def nulify_gaps(topics_metrics)
+              # Hash with all potential keys that a single sample metric can have
+              # This allows us to fill gaps not only in times but also in values
+              base_samples = topics_metrics
+                             .values
+                             .map(&:values)
+                             .flatten
+                             .select { |val| val.is_a?(Hash) }
+                             .flat_map(&:keys)
+                             .uniq
+                             .map { |key| [key, nil] }
+                             .to_h
+                             .freeze
+
+              # Normalize data in between topics reportings
+              # One topic may have a sample in a time moment when a different one does not
+              topics_metrics.each_value do |samples|
+                # All available times from all the topics
+                times = samples.values.map { |set| set.map(&:first) }.flatten.uniq
+
+                samples.each_value do |set|
+                  times.each do |time|
+                    existing_index = set.find_index { |existing_time, _| existing_time == time }
+
+                    if existing_index
+                      existing_value = set[existing_index][1]
+                      set[existing_index][1] = base_samples.merge(existing_value)
+                    else
+                      set << [time, base_samples]
+                    end
+                  end
+
+                  set.sort_by!(&:first)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/models/partition.rb

@@ -6,6 +6,33 @@ module Karafka
       module Models
         # Single topic partition data representation model
         class Partition < Lib::HashProxy
+          # @return [Symbol] one of three states in which LSO can be in the correlation to given
+          #   partition in the context of a consumer group.
+          #
+          # @note States descriptions:
+          #   - `:active` all good. No hanging transactions, processing is ok
+          #   - `:at_risk` - there may be hanging transactions but they do not affect processing
+          #     before being stuck. This means, that the transaction still may be finished
+          #     without affecting the processing, hence not having any impact.
+          #   - `:stopped` - we have reached a hanging LSO and we cannot move forward despite more
+          #     data being available. Unless the hanging transaction is killed or it finishes,
+          #     we will not move forward.
+          def lso_risk_state
+            # If last stable is falling behind the high watermark
+            if ls_offset < hi_offset
+              # But it is changing and moving fast enough, it does not mean it is stuck
+              return :active if ls_offset_fd < ::Karafka::Web.config.ui.lso_threshold
+
+              # If it is stuck but we still have work to do, this is not a tragic situation because
+              # maybe it will unstuck before we reach it
+              return :at_risk if (committed_offset || 0) < ls_offset
+
+              # If it is not changing and falling behind high, it is stuck
+              :stopped
+            else
+              :active
+            end
+          end
         end
       end
     end

data/lib/karafka/web/ui/models/process.rb

@@ -40,7 +40,7 @@ module Karafka
               .sort_by(&:started_at)
           end
 
-          # @return [Integer] collective lag on this process
+          # @return [Integer] collective stored lag on this process
           def lag_stored
             consumer_groups
               .flat_map(&:subscription_groups)
@@ -51,6 +51,17 @@ module Karafka
               .sum
           end
 
+          # @return [Integer] collective lag on this process
+          def lag
+            consumer_groups
+              .flat_map(&:subscription_groups)
+              .flat_map(&:topics)
+              .flat_map(&:partitions)
+              .map(&:lag)
+              .delete_if(&:negative?)
+              .sum
+          end
+
           # @return [Integer] number of partitions to which we are currently subscribed
           def subscribed_partitions_count
             consumer_groups