karafka-web 0.9.1 → 0.10.0.beta1

data/lib/karafka/web/ui/controllers/base_controller.rb

@@ -33,7 +33,7 @@ module Karafka
             attributes = {}
 
             scope = self.class.to_s.split('::').last.gsub(/(.)([A-Z])/, '\1_\2').downcase.gsub('_controller', '')
-            action = caller_locations(1, 1)[0].label
+            action = caller_locations(1, 1)[0].label.split('#').last
 
             instance_variables.each do |iv|
               next if iv.to_s.start_with?('@_')
@@ -66,16 +66,18 @@ module Karafka
             Responses::File.new(content, file_name)
           end
 
-          # Builds a halt 403 response
-          def deny
-            Responses::Deny.new
+          # Raises the deny error so we can render a deny block
+          # We handle this that way so we can raise this from any place we want as long as within
+          # the Roda flow and not only from controllers
+          def deny!
+            raise Errors::Ui::ForbiddenError
           end
 
           # @param resources [Hash, Array, Lib::HashProxy] object for sorting
           # @return [Hash, Array, Lib::HashProxy] sorted results
           def refine(resources)
             Lib::Sorter.new(
-              @params.sort,
+              @params.current_sort,
               allowed_attributes: self.class.sortable_attributes
             ).call(resources)
           end

data/lib/karafka/web/ui/controllers/dashboard_controller.rb

@@ -23,6 +23,14 @@ module Karafka
               @aggregated, @params.current_range
             )
 
+            @topics = Models::Metrics::Topics.new(
+              current_metrics.to_h.fetch(:consumer_groups)
+            )
+
+            @topics_charts = Models::Metrics::Charts::Topics.new(
+              @topics, @params.current_range
+            )
+
             render
           end
         end

data/lib/karafka/web/ui/controllers/requests/params.rb

@@ -24,9 +24,18 @@ module Karafka
               @request_params = request_params
             end
 
+            # @return [Hash] current search or empty if no search query present
+            def current_search
+              return @current_search if @current_search
+
+              search = @request_params['search']
+
+              @current_search = search.is_a?(Hash) ? search : {}
+            end
+
             # @return [String] sort query value
-            def sort
-              @sort ||= @request_params['sort'].to_s.downcase
+            def current_sort
+              @current_sort ||= @request_params['sort'].to_s.downcase
             end
 
             # @return [Integer] current page for paginated views
@@ -55,6 +64,11 @@ module Karafka
                 offset < -1 ? -1 : offset
               end
             end
+
+            # @return [Integer] currently selected partition or -1 if nothing provided
+            def current_partition
+              @current_partition ||= @request_params.fetch('partition', -1).to_i
+            end
           end
         end
       end

data/lib/karafka/web/ui/controllers/support_controller.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Controllers
+        # Controller to display support page
+        class SupportController < BaseController
+          # Display the support page
+          def show
+            render
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/controllers/ux_controller.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Controllers
+        # Controller to display UX components page
+        class UxController < BaseController
+          # Display the UX components page
+          def show
+            render
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/helpers/application_helper.rb

@@ -61,16 +61,16 @@ module Karafka
           #
           # @param status [String] status
           # @return [String] background style
-          def status_bg(status)
+          def status_badge(status)
             case status
-            when 'initialized' then 'bg-success'
-            when 'supervising' then 'bg-success'
-            when 'running' then 'bg-success'
-            when 'quieting' then 'bg-warning'
-            when 'quiet' then 'bg-warning text-dark'
-            when 'stopping' then 'bg-warning text-dark'
-            when 'stopped' then 'bg-danger'
-            when 'terminated' then 'bg-danger'
+            when 'initialized' then 'badge-success'
+            when 'supervising' then 'badge-success'
+            when 'running' then 'badge-success'
+            when 'quieting' then 'badge-warning'
+            when 'quiet' then 'badge-warning'
+            when 'stopping' then 'badge-warning'
+            when 'stopped' then 'badge-error'
+            when 'terminated' then 'badge-error'
             else
               raise ::Karafka::Errors::UnsupportedCaseError, status
             end
@@ -80,10 +80,10 @@ module Karafka
           #
           # @param trend [Numeric] lag trend
           # @return [String] bg classes
-          def lag_trend_bg(trend)
-            bg = 'bg-success' if trend.negative?
-            bg ||= 'bg-warning text-dark' if trend.positive?
-            bg ||= 'bg-secondary'
+          def lag_trend_badge(trend)
+            bg = 'badge-success' if trend.negative?
+            bg ||= 'badge-warning' if trend.positive?
+            bg ||= 'badge-secondary'
             bg
           end
 
@@ -93,20 +93,20 @@ module Karafka
           # @return [String] tags badges
           def tags(tags_array)
             tags_array
-              .map { |tag| %(<span class="badge bg-info">#{tag}</span>) }
+              .map { |tag| %(<span class="badge badge-info">#{tag}</span>) }
               .join(' ')
           end
 
           # Takes a kafka report state and recommends background style color
           # @param state [String] state
           # @return [String] background style
-          def kafka_state_bg(state)
+          def kafka_state_badge(state)
             case state
-            when 'up' then 'bg-success text-white'
-            when 'active' then 'bg-success text-white'
-            when 'steady' then 'bg-success text-white'
+            when 'up' then 'badge-success'
+            when 'active' then 'badge-success'
+            when 'steady' then 'badge-success'
             else
-              'bg-warning text-dark'
+              'badge-warning'
             end
           end
 
@@ -116,7 +116,7 @@ module Karafka
             return '0' if !mem_kb || mem_kb.zero?
 
             if mem_kb < 10_240
-              "#{number_with_delimiter(mem_kb)} KB"
+              "#{number_with_delimiter(mem_kb.round(4))} KB"
             elsif mem_kb < 1_000_000
               "#{number_with_delimiter((mem_kb / 1024.0).to_i)} MB"
             else
@@ -151,6 +151,27 @@ module Karafka
             %(<span title="#{stamp}">#{time}</span>)
           end
 
+          # Converts raw second count into human readable form like "12.2 minutes". etc based on
+          # number of seconds
+          #
+          # @param seconds [Numeric] number of seconds
+          # @return [String] human readable time
+          def human_readable_time(seconds)
+            case seconds
+            when 0..59
+              "#{seconds.round(2)} seconds"
+            when 60..3_599
+              minutes = seconds / 60.0
+              "#{minutes.round(2)} minutes"
+            when 3_600..86_399
+              hours = seconds / 3_600.0
+              "#{hours.round(2)} hours"
+            else
+              days = seconds / 86_400.0
+              "#{days.round(2)} days"
+            end
+          end
+
           # @param state [String] poll state
           # @param state_ch [Integer] time until next change of the poll state
           #   (from paused to active)
@@ -162,12 +183,12 @@ module Karafka
             # If state is active, there is no date of change
             if state == 'active'
               %(
-                <span class="badge #{kafka_state_bg(state)} mt-1 mb-1">#{state}</span>
+                <span class="badge #{kafka_state_badge(state)}">#{state}</span>
               )
             elsif state_ch_in_seconds > year_in_seconds
               %(
                 <span
-                  class="badge #{kafka_state_bg(state)} mt-1 mb-1"
+                  class="badge #{kafka_state_badge(state)}"
                   title="until manual resume"
                 >
                   #{state}
@@ -176,7 +197,7 @@ module Karafka
             else
               %(
                 <span
-                  class="badge #{kafka_state_bg(state)} time-title mt-1 mb-1"
+                  class="badge #{kafka_state_badge(state)} time-title"
                   title="#{Time.now + state_ch_in_seconds}"
                 >
                   #{state}
@@ -191,7 +212,7 @@ module Karafka
           def lag_with_label(lag)
             if lag.negative?
               title = 'Not available until first offset commit'
-              %(<span class="badge bg-secondary" title="#{title}">N/A</span>)
+              %(<span class="badge badge-secondary" title="#{title}">N/A</span>)
             else
               lag.to_s
             end
@@ -207,7 +228,7 @@ module Karafka
           def offset_with_label(topic_name, partition_id, offset, explore: false)
             if offset.negative?
               title = 'Not available until first offset commit'
-              %(<span class="badge bg-secondary" title="#{title}">N/A</span>)
+              %(<span class="badge badge-secondary" title="#{title}">N/A</span>)
             elsif explore
               path = explorer_path(topic_name, partition_id, offset)
               %(<a href="#{path}">#{offset}</a>)
@@ -226,29 +247,34 @@ module Karafka
             when :at_risk
               'bg-warning bg-opacity-25'
             when :stopped
-              'bg-danger bg-opacity-25'
+              'bg-error bg-opacity-25'
+            else
+              raise ::Karafka::Errors::UnsupportedCaseError
+            end
+          end
+
+          # @param details [::Karafka::Web::Ui::Models::Partition] partition information with
+          #   lso risk state info
+          # @return [String] background classes for row marking
+          def lso_risk_state_badge(details)
+            case details.lso_risk_state
+            when :active
+              ''
+            when :at_risk
+              'badge-warning'
+            when :stopped
+              'badge-error'
             else
               raise ::Karafka::Errors::UnsupportedCaseError
             end
           end
 
-          # Returns the view title html code
+          # Sets the particular page title
           #
           # @param title [String] page title
-          # @param hr [Boolean] should we add the hr tag at the end
           # @return [String] title html
-          def view_title(title, hr: false)
-            <<-HTML
-              <div class="container mb-5">
-                <div class="row">
-                  <h3>
-                    #{title}
-                  </h3>
-                </div>
-
-                #{hr ? '<hr/>' : ''}
-              </div>
-            HTML
+          def view_title(title)
+            content_for(:title) { title }
           end
 
           # @param hash [Hash] we want to flatten
@@ -295,12 +321,12 @@ module Karafka
             path = current_path(sort: desc)
             full_name = "#{name}&nbsp;#{arrow_both}"
 
-            if params.sort == desc
+            if params.current_sort == desc
               path = current_path(sort: asc)
               full_name = "#{name}&nbsp;#{rev ? arrow_up : arrow_down}"
             end
 
-            if params.sort == asc
+            if params.current_sort == asc
               path = current_path(sort: desc)
               full_name = "#{name}&nbsp;#{rev ? arrow_down : arrow_up}"
             end
@@ -336,6 +362,13 @@ module Karafka
 
             %(<span title="#{string}">#{truncated}</span>)
           end
+
+          # Renders the svg icon out of our icon set
+          # @param name [String, Symbol] name of the icon
+          # @return [String] svg icon
+          def icon(name)
+            render "shared/icons/_#{name}"
+          end
         end
       end
     end

data/lib/karafka/web/ui/helpers/paths_helper.rb

@@ -6,6 +6,30 @@ module Karafka
       module Helpers
         # Helper for web ui paths builders
         module PathsHelper
+          # Helper method to flatten nested hashes and arrays
+          # @param prefix [String] The prefix for nested keys, initially an empty string.
+          # @param hash [Hash, Array] The nested hash or array to be flattened.
+          # @param [Hash] result The hash to store the flattened key-value pairs.
+          # @return [Hash] The flattened hash with keys in bracket notation suitable for URL
+          #   encoding.
+          def flatten_params(prefix, hash, result = {})
+            if hash.is_a?(Hash)
+              hash.each do |k, v|
+                new_prefix = prefix.empty? ? k.to_s : "#{prefix}[#{k}]"
+                flatten_params(new_prefix, v, result)
+              end
+            elsif hash.is_a?(Array)
+              hash.each_with_index do |v, i|
+                new_prefix = "#{prefix}[#{i}]"
+                flatten_params(new_prefix, v, result)
+              end
+            else
+              result[prefix] = hash.to_s
+            end
+
+            result
+          end
+
           # Generates a full path with the root path out of the provided arguments
           #
           # @param args [Array<String, Numeric>] arguments that will make the path

data/lib/karafka/web/ui/helpers/tailwind_helper.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Helpers
+        # Helper for generating tailwind rendering components with Ruby
+        # Simplifies many places in the UI
+        module TailwindHelper
+          # style types of components we support
+          TYPES = %i[
+            info
+            error
+            warning
+            success
+            primary
+            secondary
+          ].freeze
+
+          # @return [Array<Symbol>] style types of components we support
+          def tailwind_types
+            TYPES
+          end
+
+          # Renders a plain badge
+          # @param content [String] badge content
+          # @param classes [String] extra css classes
+          # @return [String] badge html
+          def badge(content, classes: '')
+            %(<span class="badge #{classes}">#{content}</span>)
+          end
+
+          # Renders a link to with button styling
+          # @param name [String] button name
+          # @param path [String] path to where to go
+          # @param classes [String] extra css classes
+          # @return [String] button link html
+          def link_button(name, path, classes: '')
+            %(<a href="#{path}" class="btn #{classes}">#{name}</a>)
+          end
+
+          # Defines various methods for badges and links that simplify defining them without
+          # having to provide whole classes scopes always.
+          TYPES.each do |type|
+            define_method :"badge_#{type}" do |content, classes: ''|
+              badge(content, classes: "#{classes} badge-#{type}")
+            end
+
+            define_method :"badge_#{type}_sm" do |content, classes: ''|
+              badge(content, classes: "#{classes} badge-#{type} badge-sm")
+            end
+
+            define_method :"link_button_#{type}" do |name, path, classes: ''|
+              link_button(name, path, classes: "#{classes} btn-#{type}")
+            end
+
+            define_method :"link_button_#{type}_sm" do |name, path, classes: ''|
+              link_button(name, path, classes: "#{classes} btn-#{type} btn-sm")
+            end
+
+            # @param message [String] alert message
+            # @return [String] html with alert
+            define_method :"alert_#{type}" do |message|
+              partial(
+                "shared/alerts/#{type}",
+                locals: {
+                  message: message
+                }
+              )
+            end
+
+            # @param message [String] alert message
+            # @return [String] html with alert
+            define_method :"alert_box_#{type}" do |title, description = nil, &block|
+              description = capture_erb(&block) if block
+
+              inject_erb partial(
+                "shared/alerts/box_#{type}",
+                locals: {
+                  title: title,
+                  description: description
+                }
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/lib/sorter.rb

@@ -33,7 +33,7 @@ module Karafka
             @field = '' unless allowed_attributes.include?(@field)
 
             # Things we have already seen and sorted. Prevents crashing on the circular
-            # dependencies sorting when same resources are present in different parts of the three
+            # dependencies sorting when same resources are present in different parts of the tree
             @seen = {}
           end
 

data/lib/karafka/web/ui/models/metrics/aggregated.rb

@@ -24,6 +24,7 @@ module Karafka
             #
             # If at least two elements do not exist for given delta range, we keep it empty
             DELTA_KEYS = %i[
+              jobs
               batches
               messages
               errors

data/lib/karafka/web/ui/models/metrics/charts/topics.rb

@@ -47,48 +47,64 @@ module Karafka
                 per_topic.merge('total sum' => total.to_a).to_json
               end
 
-              # @return [String] JSON with producers pace that represents high-watermarks sum for
-              #   each topic
-              def topics_pace
-                topics = {}
+              # @return [String] JSON with per-topic, highest LSO freeze duration. Useful for
+              #   debugging of issues arising from hanging transactions
+              def max_lso
+                topics = Hash.new { |h, k| h[k] = Hash.new { |h2, k2| h2[k2] = [] } }
 
                 @data.to_h.each do |topic, metrics|
                   topic_without_cg = topic.split('[').first
 
-                  # If we've already seen this topic data, we can skip
-                  next if topics.include?(topic_without_cg)
+                  metrics.each do |current|
+                    ls_offset_fd = current.last[:ls_offset_fd] || 0
 
-                  topics[topic_without_cg] = metrics.map do |current|
-                    [current.first, current.last[:pace]]
+                    # We convert this to seconds from milliseconds due to our Web UI precision
+                    # Reporting is in ms for consistency
+                    normalized_fd = (ls_offset_fd / 1_000.0).round
+
+                    topics[topic_without_cg][current.first] << normalized_fd
                   end
                 end
 
                 topics.each_value(&:compact!)
+                topics.each_value { |metrics| metrics.transform_values!(&:max) }
+                topics.transform_values! { |values| values.to_a.sort_by!(&:first) }
                 topics.to_json
               end
 
-              # @return [String] JSON with per-topic, highest LSO freeze duration. Useful for
-              #   debugging of issues arising from hanging transactions
-              def max_lso_time
-                topics = Hash.new { |h, k| h[k] = Hash.new { |h2, k2| h2[k2] = [] } }
+              # @return [String] JSON with producers pace that represents high-watermarks sum for
+              #   each topic.
+              #
+              # @note There is a case where data reported (sum on a topic) is lower then the
+              #   previous value. This can happen around rebalances because consumer may not
+              #   have all watermark offsets reported. This may cause consumers not to report some
+              #   of the partitions, effectively lowering the sum. Since high-watermark offsets can
+              #   only move forward, we compensate this by assuming that a lower value than
+              #   previous is an artefact of that type and we replace it with the max value we had
+              #   effectively compensating for under-reporting
+              def pace
+                topics = {}
 
                 @data.to_h.each do |topic, metrics|
                   topic_without_cg = topic.split('[').first
 
-                  metrics.each do |current|
-                    ls_offset_fd = current.last[:ls_offset_fd] || 0
+                  # If we've already seen this topic data, we can skip
+                  next if topics.include?(topic_without_cg)
 
-                    # We convert this to seconds from milliseconds due to our Web UI precision
-                    # Reporting is in ms for consistency
-                    normalized_fd = (ls_offset_fd / 1_000.0).round
+                  max_pace = 0
 
-                    topics[topic_without_cg][current.first] << normalized_fd
+                  topics[topic_without_cg] = metrics.map do |current|
+                    # Pace may be empty when for a given moment in time we got no info on
+                    # one of the topics. In such case we can compensate with max or 0
+                    current_pace = current.last[:pace] || 0
+
+                    max_pace = current_pace if current_pace > max_pace
+
+                    [current.first, max_pace]
                   end
                 end
 
                 topics.each_value(&:compact!)
-                topics.each_value { |metrics| metrics.transform_values!(&:max) }
-                topics.transform_values! { |values| values.to_a.sort_by!(&:first) }
                 topics.to_json
               end
             end

data/lib/karafka/web/ui/models/status.rb

@@ -209,10 +209,37 @@ module Karafka
             )
           end
 
+          # @return [Status::Step] Is there a significant lag in the reporting of aggregated data
+          #   back to the Kafka. If yes, it means that the results in the Web UI will be delayed
+          #   against the reality. Often it means, that there is over-saturation on the consumer
+          #   that is materializing the states.
+          #
+          # @note Since both states and metrics are reported together, it is enough for us to check
+          #   on one of them.
+          def materializing_lag
+            max_lag = (Web.config.tracking.interval * 2) / 1_000
+
+            details = { lag: 0, max_lag: max_lag }
+
+            status = if live_reporting.success?
+                       lag = Time.now.to_f - @current_state.dispatched_at
+                       details[:lag] = lag
+
+                       lag > max_lag ? :failure : :success
+                     else
+                       :halted
+                     end
+
+            Step.new(
+              status,
+              details
+            )
+          end
+
           # @return [Status::Step] is there a subscription to our reports topic that is being
           #   consumed actively.
           def state_calculation
-            if live_reporting.success?
+            if materializing_lag.success?
               @subscriptions ||= Models::Health
                                  .current(@current_state)
                                  .values.map { |consumer_group| consumer_group[:topics] }

data/lib/karafka/web/ui/public/images/calendar.svg

@@ -0,0 +1,3 @@
+<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="size-6">
+  <path stroke-linecap="round" stroke-linejoin="round" d="M6.75 3v2.25M17.25 3v2.25M3 18.75V7.5a2.25 2.25 0 0 1 2.25-2.25h13.5A2.25 2.25 0 0 1 21 7.5v11.25m-18 0A2.25 2.25 0 0 0 5.25 21h13.5A2.25 2.25 0 0 0 21 18.75m-18 0v-7.5A2.25 2.25 0 0 1 5.25 9h13.5A2.25 2.25 0 0 1 21 11.25v7.5m-9-6h.008v.008H12v-.008ZM12 15h.008v.008H12V15Zm0 2.25h.008v.008H12v-.008ZM9.75 15h.008v.008H9.75V15Zm0 2.25h.008v.008H9.75v-.008ZM7.5 15h.008v.008H7.5V15Zm0 2.25h.008v.008H7.5v-.008Zm6.75-4.5h.008v.008h-.008v-.008Zm0 2.25h.008v.008h-.008V15Zm0 2.25h.008v.008h-.008v-.008Zm2.25-4.5h.008v.008H16.5v-.008Zm0 2.25h.008v.008H16.5V15Z" />
+</svg>