karafka-web 0.8.2 → 0.9.0.rc2

data/lib/karafka/web/ui/helpers/application_helper.rb

@@ -8,6 +8,29 @@ module Karafka
       module Helpers
         # Main application helper
         module ApplicationHelper
+          # Default attribute names mapped from the attributes themselves
+          # It makes it easier as we do not have to declare those all the time
+          SORT_NAMES = {
+            id: 'ID',
+            partition_id: 'Partition',
+            memory_usage: 'RSS',
+            started_at: 'Started',
+            committed_offset: 'Committed',
+            last_offset: 'Last',
+            first_offset: 'First',
+            lo_offset: 'Low',
+            hi_offset: 'High',
+            ls_offset: 'LSO',
+            lag_hybrid: 'Lag',
+            lag_stored: 'Stored',
+            stored_offset: 'Stored',
+            fetch_state: 'Fetch',
+            poll_state: 'Poll',
+            lso_risk_state: 'LSO'
+          }.freeze
+
+          private_constant :SORT_NAMES
+
           # Adds active class to the current location in the nav if needed
           # @param location [Hash]
           def nav_class(location)
@@ -260,7 +283,7 @@ module Karafka
           def sort_link(name, attribute = nil, rev: false)
             unless attribute
               attribute = name
-              name = attribute.to_s.tr('_', ' ').capitalize
+              name = SORT_NAMES[attribute] || attribute.to_s.tr('_', ' ').tr('?', '').capitalize
             end
 
             arrow_both = '⇕'
@@ -284,6 +307,35 @@ module Karafka
 
             "<a class=\"sort\" href=\"#{path}\">#{full_name}</a>"
           end
+
+          # Truncates given text if it is too long and wraps it with a title with full text.
+          # Can use a middle-based strategy that keeps beginning and ending of a string instead of
+          # keeping just the beginning.
+          #
+          # The `:middle` strategy is useful when we have strings such as really long process names
+          # that have important beginning and end but middle can be removed without risk of not
+          # allowing user to recognize the content.
+          #
+          # @param string [String] string we want to truncate
+          # @param length [Integer] max length of the final string that we accept before truncating
+          # @param omission [String] truncation omission
+          # @param strategy [Symbol] `:default` or `:middle` how should we truncate
+          # @return [String] HTML span tag with truncated content and full content title
+          def truncate(string, length: 50, omission: '...', strategy: :default)
+            return string if string.length <= length
+
+            case strategy
+            when :default
+              truncated = string[0...(length - omission.length)] + omission
+            when :middle
+              part_length = (length - omission.length) / 2
+              truncated = string[0...part_length] + omission + string[-part_length..]
+            else
+              raise Karafka::Errors::UnsupportedCaseError, "Unknown strategy: #{strategy}"
+            end
+
+            %(<span title="#{string}">#{truncated}</span>)
+          end
         end
       end
     end

data/lib/karafka/web/ui/lib/paginations/offset_based.rb

@@ -42,13 +42,12 @@ module Karafka
             #   not consider it as a first page and we allow to "reset" to -1 via the first page
             #   button
             def first_offset?
-              @current_offset != -1
+              @current_offset != -1 && @previous_offset != false
             end
 
-            # @return [Boolean] first page offset is always nothing because we use the default -1
-            #   for the offset.
+            # @return [Integer] -1 because it will then select the highest offsets
             def first_offset
-              false
+              -1
             end
 
             # @return [Boolean] Active previous page link when it is not the first page

data/lib/karafka/web/ui/lib/safe_runner.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Lib
+        # Class used to execute code that can fail but we do not want to fail the whole operation.
+        # The primary use-case is for displaying deserialized data. We always need to assume, that
+        # part of the data can be corrupted and it should not crash the whole UI.
+        #
+        # It caches the result and does not run the code twice (only once)
+        class SafeRunner
+          attr_reader :error, :result
+
+          # @param block [Proc] code we want to safe-guard
+          def initialize(&block)
+            @code = block
+            @executed = false
+            @success = false
+            @error = nil
+            @result = nil
+          end
+
+          # @return [Boolean] was the code execution successful or not
+          def success?
+            return @success if executed?
+
+            call
+
+            @success
+          end
+
+          # @return [Boolean] was the code execution failed or not
+          def failure?
+            !success?
+          end
+
+          # Runs the execution and returns block result
+          def call
+            return @result if executed?
+
+            @executed = true
+            @result = @code.call
+            @success = true
+            @result
+          rescue StandardError => e
+            @error = e
+            @success = false
+          end
+
+          # @return [Boolean] was the code executed already or not yet
+          def executed?
+            @executed
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/models/broker.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Models
+        # Represents a single broker data within the cluster
+        class Broker < Lib::HashProxy
+          class << self
+            # @return [Array<Broker>] all brokers in the cluster
+            def all
+              # We do not cache here because we want the most recent state of brokers possible
+              ClusterInfo.fetch(cached: false).brokers.map do |broker|
+                new(broker)
+              end
+            end
+
+            # Finds requested broker
+            #
+            # @param broker_id [String, Integer] id of the broker
+            # @return [Broker]
+            # @raise [::Karafka::Web::Errors::Ui::NotFoundError]
+            def find(broker_id)
+              found = all.find { |broker| broker.id.to_s == broker_id }
+
+              return found if found
+
+              raise(::Karafka::Web::Errors::Ui::NotFoundError, broker_id)
+            end
+          end
+
+          # @return [Integer]
+          def id
+            broker_id
+          end
+
+          # @return [String]
+          def name
+            broker_name
+          end
+
+          # @return [Integer]
+          def port
+            broker_port
+          end
+
+          # @return [String] full broker name for presentation
+          def full_name
+            "#{id} - #{name}:#{port}"
+          end
+
+          # @return [Array<Karafka::Admin::Configs::Config>] all broker configs
+          def configs
+            # We copy the array because the result one is frozen and we sort
+            @configs ||= ::Karafka::Admin::Configs.describe(
+              ::Karafka::Admin::Configs::Resource.new(
+                type: :broker,
+                name: id
+              )
+            ).first.configs.dup
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/models/health.rb

@@ -8,7 +8,7 @@ module Karafka
         class Health
           class << self
             # @param state [State] current system state
-            # @return [Hash] has with aggregated statistics
+            # @return [Hash] hash with aggregated statistics
             def current(state)
               stats = {}
 
@@ -18,6 +18,32 @@ module Karafka
               sort_structure(stats)
             end
 
+            # @return [Hash] hash with cluster lag data
+            def cluster_lags_with_offsets
+              # We need to remap raw results so they comply with our sorting flows
+              mapped_lags = {}
+
+              ::Karafka::Admin.read_lags_with_offsets(
+                active_topics_only: Web.config.ui.visibility.active_topics_cluster_lags_only
+              ).each do |consumer_group, topics|
+                mapped_lags[consumer_group] ||= {}
+
+                topics.each do |topic_name, partitions_details|
+                  mapped_lags[consumer_group][topic_name] ||= []
+
+                  partitions_details.each do |partition_id, lags_with_offsets|
+                    mapped_lags[consumer_group][topic_name] << {
+                      id: partition_id,
+                      lag: lags_with_offsets.fetch(:lag),
+                      stored_offset: lags_with_offsets.fetch(:offset)
+                    }
+                  end
+                end
+              end
+
+              mapped_lags
+            end
+
             private
 
             # Aggregates data on a per topic basis (in the context of a consumer group)
@@ -62,7 +88,7 @@ module Karafka
             #
             # @param state [State]
             def iterate_partitions(state)
-              # By default processes are sort by name and this is not what we want here
+              # By default processes are sort by id and this is not what we want here
               # We want to make sure that the newest data is processed the last, so we get
               # the most accurate state in case of deployments and shutdowns, etc without the
               # expired processes partitions data overwriting the newly created processes

data/lib/karafka/web/ui/models/message.rb

@@ -100,10 +100,16 @@ module Karafka
 
                 previous_offset = start_offset + count
 
+                if previous_offset >= high_offset
+                  previous_offset = false
+                elsif previous_offset + (per_page - 1) > high_offset
+                  previous_offset = high_offset - per_page
+                else
+                  previous_offset
+                end
+
                 return [
-                  # If there is a potential previous page with more recent data, compute its
-                  # offset
-                  previous_offset >= high_offset ? false : previous_offset,
+                  previous_offset,
                   fill_compacted(messages, partition_id, context_offset, context_count, high_offset).reverse,
                   next_offset
                 ]

data/lib/karafka/web/ui/models/process.rb

@@ -18,11 +18,6 @@ module Karafka
             end
           end
 
-          # @return [String] process id without the name and ip
-          def id
-            @id ||= name.split(':').last
-          end
-
           # @return [Array<ConsumerGroup>] consumer groups to which this process is subscribed in
           #   an alphabetical order
           def consumer_groups
@@ -41,6 +36,16 @@ module Karafka
               .then { |jobs| Jobs.new(jobs) }
           end
 
+          # @return [Integer] number of running jobs on a process
+          def running_jobs_count
+            jobs.running.count
+          end
+
+          # @return [Integer] number of pending jobs on a process
+          def pending_jobs_count
+            jobs.pending.count
+          end
+
           # @return [Integer] collective hybrid lag on this process
           def lag_hybrid
             consumer_groups

data/lib/karafka/web/ui/models/processes.rb

@@ -79,11 +79,11 @@ module Karafka
               end
             end
 
-            # Ensures that we always return processes sorted by their name
+            # Ensures that we always return processes sorted by their id
             # @param processes [Array<Hash>]
             # @return [Array<Hash>] sorted processes data
             def sort_processes(processes)
-              processes.sort_by { |consumer| consumer[:process][:name] }
+              processes.sort_by { |consumer| consumer[:process][:id] }
             end
           end
         end

data/lib/karafka/web/ui/models/status.rb

@@ -44,7 +44,7 @@ module Karafka
           # aware of the deserializer, etc
           def enabled
             enabled = ::Karafka::App.routes.map(&:name).include?(
-              ::Karafka::Web.config.processing.consumer_group
+              ::Karafka::Web.config.group_id
             )
 
             Step.new(

data/lib/karafka/web/ui/models/topic.rb

@@ -6,6 +6,28 @@ module Karafka
       module Models
         # Single topic data representation model
         class Topic < Lib::HashProxy
+          class << self
+            # @return [Array<Broker>] all topics in the cluster
+            def all
+              ClusterInfo.fetch.topics.map do |topic|
+                new(topic)
+              end
+            end
+
+            # Finds requested topic
+            #
+            # @param topic_name [String] name of the topic
+            # @return [Topic]
+            # @raise [::Karafka::Web::Errors::Ui::NotFoundError]
+            def find(topic_name)
+              found = all.find { |topic| topic.topic_name == topic_name }
+
+              return found if found
+
+              raise(::Karafka::Web::Errors::Ui::NotFoundError, topic_name)
+            end
+          end
+
           # @return [Array<Partition>] All topic partitions data
           def partitions
             super.map do |partition_id, partition_hash|
@@ -14,6 +36,62 @@ module Karafka
               Partition.new(partition_hash)
             end
           end
+
+          # @return [Array<Karafka::Admin::Configs::Config>] all topic configs
+          def configs
+            @configs ||= ::Karafka::Admin::Configs.describe(
+              ::Karafka::Admin::Configs::Resource.new(
+                type: :topic,
+                name: topic_name
+              )
+            ).first.configs.dup
+          end
+
+          # Generates info about estimated messages distribution in partitions, allowing for
+          # inspection and detection of imbalances
+          #
+          # @param partitions [Array<Integer>] partitions we're interested in
+          #
+          # @return [Array<HashProxy, Array<HashProxy>>] array where first value contains
+          #   aggregated statistics and then the second value is an array with per partition data
+          def distribution(partitions)
+            sum = 0.0
+            avg = 0.0
+
+            counts = partitions.map do |partition_id|
+              offsets = Admin.read_watermark_offsets(topic_name, partition_id)
+              count = offsets.last - offsets.first
+
+              sum += count
+
+              {
+                count: count,
+                partition_id: partition_id
+              }
+            end
+
+            avg = sum / counts.size
+
+            counts.each do |part_stats|
+              count = part_stats[:count]
+
+              part_stats[:share] = ((count / sum) * 100).round(2)
+              part_stats[:diff] = ((count - avg) / avg) * 100
+            end
+
+            variance = counts
+                       .map { |part_stats| part_stats[:count] }
+                       .sum { |count| (count - avg)**2 } / counts.size
+
+            std_dev = Math.sqrt(variance)
+            std_dev_rel = ((std_dev / avg) * 100).round(2)
+
+            [
+              # round stdev since its message count
+              Lib::HashProxy.new(std_dev: std_dev.round, std_dev_rel: std_dev_rel, sum: sum),
+              counts.map { |part_stats| Lib::HashProxy.new(part_stats) }
+            ]
+          end
         end
       end
     end

data/lib/karafka/web/ui/public/javascripts/application.js

@@ -18,6 +18,18 @@ function updateTimeAgo() {
   }
 }
 
+// Cheap way to do breadcrumbs
+function refreshTitle() {
+  const breadcrumbs = document.querySelectorAll('.breadcrumb a');
+  let breadcrumbTexts = Array.from(breadcrumbs).slice(1).map(crumb => crumb.textContent.trim());
+
+  if (breadcrumbTexts.length > 0) {
+    document.title = breadcrumbTexts.join(' > ') + ' - Karafka Web UI';
+  } else {
+    document.title = 'Karafka Web UI';
+  }
+}
+
 // To prevent from flickering, the UI is initially hidden and visible when all the JS components
 // are fully initialized
 function displayUi() {
@@ -66,11 +78,16 @@ function addListeners() {
   hljs.highlightAll();
   updateTimeAgo();
   redirectToPartition();
-  manageTabs();
+
+  const tabsManager = new TabsManager();
+  tabsManager.manageTabs();
+
   manageCharts();
   bindActionsConfirmations();
   loadOffsetLookupDatePicker();
   displayUi();
+
+  refreshTitle()
 }
 
 var ready = (callback) => {

data/lib/karafka/web/ui/public/javascripts/charts/data_formatting_utility.js

@@ -0,0 +1,71 @@
+const DataFormattingUtils = {
+  niceBytes(x, precision = 2) {
+    const units = ['bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
+    let l = 0, n = parseInt(x, 10) || 0;
+
+    while (n >= 1024 && ++l) {
+      n /= 1024;
+    }
+
+    return `${n.toFixed(n < 10 && l > 0 ? 1 : precision)} ${units[l]}`;
+  },
+
+  formatLabelX(value, type) {
+    switch (type) {
+      case 'date':
+        let date = new Date(value * 1000)
+        let date_str =
+          ("00" + (date.getMonth() + 1)).slice(-2) + "/" +
+          ("00" + date.getDate()).slice(-2) + "/" +
+          date.getFullYear() + " " +
+          ("00" + date.getHours()).slice(-2) + ":" +
+          ("00" + date.getMinutes()).slice(-2) + ":" +
+          ("00" + date.getSeconds()).slice(-2);
+
+        return date_str
+      default:
+        return value
+    }
+  },
+
+  formatTooltip(type, tooltipItem) {
+    let value = tooltipItem.parsed.y;
+    let label = tooltipItem.dataset.label;
+
+    switch(type) {
+    case 'percentage':
+      if (Math.floor(value) === value) {
+        return label + ': ' + value + ' %';
+      } else {
+        return label + ': ' + (Math.round(value * 100) / 100) + ' %';
+      }
+    case 'memory':
+      return label + ': ' + DataFormattingUtils.niceBytes(value * 1024, 2);
+    default:
+      return tooltipItem.yLabel
+    }
+  },
+
+  formatLabelY(type, label) {
+    switch(type) {
+    case 'percentage':
+      if (Math.floor(label) === label) {
+        return label + '%'
+      } else {
+        return (Math.round(label * 100) / 100) + '%'
+      }
+    case 'memory':
+      return DataFormattingUtils.niceBytes(label * 1024, 1)
+    default:
+      if (Math.floor(label) === label) {
+        return label
+      } else {
+        return Math.round(label * 100) / 100
+      }
+    }
+  },
+
+  isFractionalPrecision(value) {
+    return value !== Math.floor(value);
+  }
+};

data/lib/karafka/web/ui/public/javascripts/charts/dataset_state_manager.js

@@ -0,0 +1,49 @@
+class DatasetStateManager {
+  constructor() {
+    this.storageKey = 'karafkaDisabledDatasets';
+  }
+
+  // Reads all disabled datasets from localStorage
+  readAll() {
+    const raw = localStorage.getItem(this.storageKey);
+    return raw ? JSON.parse(raw) : {};
+  }
+
+  // Saves all disabled datasets to localStorage
+  saveAll(data) {
+    localStorage.setItem(this.storageKey, JSON.stringify(data));
+  }
+
+  // Saves the current disabled datasets for all '.chartjs-line' charts
+  saveCurrent() {
+    const charts = document.querySelectorAll('.chartjs');
+    const url = window.location.href.split('?')[0];
+    let currentDisabled = {};
+    let allDisabled = this.readAll();
+
+    charts.forEach(chart => {
+      const chartId = chart.id;
+      const chartInstance = Chart.getChart(chartId);
+      if (!chartInstance || !chartInstance.legend || !chartInstance.legend.legendItems) return;
+
+      let disabledIndices = chartInstance.legend.legendItems
+        .map((item, index) => item.hidden ? index : null)
+        .filter(index => index !== null);
+
+      if (disabledIndices.length > 0) {
+        currentDisabled[chartId] = disabledIndices;
+      }
+    });
+
+    allDisabled[url] = currentDisabled;
+    this.saveAll(allDisabled);
+  }
+
+  // Retrieves the disabled datasets for a specific chart ID
+  getCurrentChart(chartId) {
+    const url = window.location.href.split('?')[0];
+    let allDisabled = this.readAll();
+    let currentDisabled = allDisabled[url] || {};
+    return currentDisabled[chartId] || [];
+  }
+}