karafka-web 0.9.1 → 0.10.0.rc1

data/lib/karafka/web/ui/base.rb

@@ -7,7 +7,7 @@ module Karafka
       class Base < Roda
         include Helpers::PathsHelper
         include Helpers::ApplicationHelper
-        include Helpers::AlertsHelper
+        include Helpers::TailwindHelper
 
         # Details that need to be evaluated in the context of OSS or Pro web UI.
         # If those would be evaluated in the base, they would not be initialized as expected
@@ -17,7 +17,9 @@ module Karafka
             root: Karafka::Web.gem_root.join('lib/karafka/web/ui/public'),
             # Cache all static files for the end user for as long as possible
             # We can do it because we ship per version assets so they invalidate with gem bumps
-            headers: { 'Cache-Control' => 'max-age=31536000, immutable' }
+            headers: { 'Cache-Control' => 'max-age=31536000, immutable' },
+            gzip: true,
+            brotli: true
           )
           plugin :render_each
           plugin :partials
@@ -35,6 +37,9 @@ module Karafka
         plugin :hooks
         plugin :flash
         plugin :path
+        plugin :capture_erb
+        plugin :content_for
+        plugin :inject_erb
 
         # Based on
         # https://github.com/sidekiq/sidekiq/blob/ae6ca119/lib/sidekiq/web/application.rb#L8
@@ -48,7 +53,7 @@ module Karafka
           csp.manifest_src "'self'"
           csp.media_src "'self'"
           csp.object_src "'none'"
-          csp.script_src "'self' https: http: 'unsafe-inline'"
+          csp.script_src "'self' https: http:"
           csp.style_src "'self' https: http: 'unsafe-inline'"
           csp.worker_src "'self'"
           csp.base_uri "'self'"
@@ -60,12 +65,6 @@ module Karafka
           render_response(result)
         end
 
-        handle_block_result Controllers::Responses::Deny do
-          @error = true
-          response.status = 403
-          view 'shared/exceptions/not_allowed'
-        end
-
         # Redirect either to referer back or to the desired path
         handle_block_result Controllers::Responses::Redirect do |result|
           # Map redirect flashes (if any) to Roda flash messages
@@ -84,13 +83,18 @@ module Karafka
         plugin :error_handler, classes: [
           ::Rdkafka::RdkafkaError,
           Errors::Ui::NotFoundError,
-          Errors::Ui::ProOnlyError
+          Errors::Ui::ProOnlyError,
+          Errors::Ui::ForbiddenError
         ] do |e|
           @error = true
 
-          if e.is_a?(Errors::Ui::ProOnlyError)
+          case e
+          when Errors::Ui::ProOnlyError
             response.status = 402
             view 'shared/exceptions/pro_only'
+          when Errors::Ui::ForbiddenError
+            response.status = 403
+            view 'shared/exceptions/not_allowed'
           else
             response.status = 404
             view 'shared/exceptions/not_found'
@@ -128,15 +132,17 @@ module Karafka
         # the query data. Query data takes priority over request params.
         # @param query_data [Hash] query params we want to add to the current path
         path :current do |query_data = {}|
-          q = query_data
-              .transform_values(&:to_s)
-              .transform_keys(&:to_s)
-              .then { |candidates| request.params.merge(candidates) }
-              .select { |_, v| v }
-              .map { |k, v| "#{k}=#{CGI.escape(v.to_s)}" }
-              .join('&')
-
-          [request.path, q].compact.delete_if(&:empty?).join('?')
+          # Merge existing request parameters with new query data
+          merged_params = request.params.deep_merge(query_data)
+
+          # Flatten the merged parameters
+          flattened_params = flatten_params('', merged_params)
+
+          # Build the query string from the flattened parameters
+          query_string = URI.encode_www_form(flattened_params)
+
+          # Construct the full path with query string
+          [request.path, query_string].compact.join('?')
         end
 
         # Sets appropriate template variables based on the response object and renders the

data/lib/karafka/web/ui/controllers/base_controller.rb

@@ -33,7 +33,7 @@ module Karafka
             attributes = {}
 
             scope = self.class.to_s.split('::').last.gsub(/(.)([A-Z])/, '\1_\2').downcase.gsub('_controller', '')
-            action = caller_locations(1, 1)[0].label
+            action = caller_locations(1, 1)[0].label.split('#').last
 
             instance_variables.each do |iv|
               next if iv.to_s.start_with?('@_')
@@ -66,16 +66,18 @@ module Karafka
             Responses::File.new(content, file_name)
           end
 
-          # Builds a halt 403 response
-          def deny
-            Responses::Deny.new
+          # Raises the deny error so we can render a deny block
+          # We handle this that way so we can raise this from any place we want as long as within
+          # the Roda flow and not only from controllers
+          def deny!
+            raise Errors::Ui::ForbiddenError
           end
 
           # @param resources [Hash, Array, Lib::HashProxy] object for sorting
           # @return [Hash, Array, Lib::HashProxy] sorted results
           def refine(resources)
             Lib::Sorter.new(
-              @params.sort,
+              @params.current_sort,
               allowed_attributes: self.class.sortable_attributes
             ).call(resources)
           end

data/lib/karafka/web/ui/controllers/dashboard_controller.rb

@@ -23,6 +23,14 @@ module Karafka
               @aggregated, @params.current_range
             )
 
+            @topics = Models::Metrics::Topics.new(
+              current_metrics.to_h.fetch(:consumer_groups)
+            )
+
+            @topics_charts = Models::Metrics::Charts::Topics.new(
+              @topics, @params.current_range
+            )
+
             render
           end
         end

data/lib/karafka/web/ui/controllers/requests/params.rb

@@ -24,9 +24,18 @@ module Karafka
               @request_params = request_params
             end
 
+            # @return [Hash] current search or empty if no search query present
+            def current_search
+              return @current_search if @current_search
+
+              search = @request_params['search']
+
+              @current_search = search.is_a?(Hash) ? search : {}
+            end
+
             # @return [String] sort query value
-            def sort
-              @sort ||= @request_params['sort'].to_s.downcase
+            def current_sort
+              @current_sort ||= @request_params['sort'].to_s.downcase
             end
 
             # @return [Integer] current page for paginated views
@@ -55,6 +64,11 @@ module Karafka
                 offset < -1 ? -1 : offset
               end
             end
+
+            # @return [Integer] currently selected partition or -1 if nothing provided
+            def current_partition
+              @current_partition ||= @request_params.fetch('partition', -1).to_i
+            end
           end
         end
       end

data/lib/karafka/web/ui/controllers/support_controller.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Controllers
+        # Controller to display support page
+        class SupportController < BaseController
+          # Display the support page
+          def show
+            render
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/controllers/ux_controller.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Controllers
+        # Controller to display UX components page
+        class UxController < BaseController
+          # Display the UX components page
+          def show
+            render
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/helpers/application_helper.rb

@@ -61,16 +61,16 @@ module Karafka
           #
           # @param status [String] status
           # @return [String] background style
-          def status_bg(status)
+          def status_badge(status)
             case status
-            when 'initialized' then 'bg-success'
-            when 'supervising' then 'bg-success'
-            when 'running' then 'bg-success'
-            when 'quieting' then 'bg-warning'
-            when 'quiet' then 'bg-warning text-dark'
-            when 'stopping' then 'bg-warning text-dark'
-            when 'stopped' then 'bg-danger'
-            when 'terminated' then 'bg-danger'
+            when 'initialized' then 'badge-success'
+            when 'supervising' then 'badge-success'
+            when 'running' then 'badge-success'
+            when 'quieting' then 'badge-warning'
+            when 'quiet' then 'badge-warning'
+            when 'stopping' then 'badge-warning'
+            when 'stopped' then 'badge-error'
+            when 'terminated' then 'badge-error'
             else
               raise ::Karafka::Errors::UnsupportedCaseError, status
             end
@@ -80,10 +80,10 @@ module Karafka
           #
           # @param trend [Numeric] lag trend
           # @return [String] bg classes
-          def lag_trend_bg(trend)
-            bg = 'bg-success' if trend.negative?
-            bg ||= 'bg-warning text-dark' if trend.positive?
-            bg ||= 'bg-secondary'
+          def lag_trend_badge(trend)
+            bg = 'badge-success' if trend.negative?
+            bg ||= 'badge-warning' if trend.positive?
+            bg ||= 'badge-secondary'
             bg
           end
 
@@ -93,20 +93,20 @@ module Karafka
           # @return [String] tags badges
           def tags(tags_array)
             tags_array
-              .map { |tag| %(<span class="badge bg-info">#{tag}</span>) }
+              .map { |tag| %(<span class="badge badge-info">#{tag}</span>) }
               .join(' ')
           end
 
           # Takes a kafka report state and recommends background style color
           # @param state [String] state
           # @return [String] background style
-          def kafka_state_bg(state)
+          def kafka_state_badge(state)
             case state
-            when 'up' then 'bg-success text-white'
-            when 'active' then 'bg-success text-white'
-            when 'steady' then 'bg-success text-white'
+            when 'up' then 'badge-success'
+            when 'active' then 'badge-success'
+            when 'steady' then 'badge-success'
             else
-              'bg-warning text-dark'
+              'badge-warning'
             end
           end
 
@@ -116,7 +116,7 @@ module Karafka
             return '0' if !mem_kb || mem_kb.zero?
 
             if mem_kb < 10_240
-              "#{number_with_delimiter(mem_kb)} KB"
+              "#{number_with_delimiter(mem_kb.round(4))} KB"
             elsif mem_kb < 1_000_000
               "#{number_with_delimiter((mem_kb / 1024.0).to_i)} MB"
             else
@@ -151,6 +151,27 @@ module Karafka
             %(<span title="#{stamp}">#{time}</span>)
           end
 
+          # Converts raw second count into human readable form like "12.2 minutes". etc based on
+          # number of seconds
+          #
+          # @param seconds [Numeric] number of seconds
+          # @return [String] human readable time
+          def human_readable_time(seconds)
+            case seconds
+            when 0..59
+              "#{seconds.round(2)} seconds"
+            when 60..3_599
+              minutes = seconds / 60.0
+              "#{minutes.round(2)} minutes"
+            when 3_600..86_399
+              hours = seconds / 3_600.0
+              "#{hours.round(2)} hours"
+            else
+              days = seconds / 86_400.0
+              "#{days.round(2)} days"
+            end
+          end
+
           # @param state [String] poll state
           # @param state_ch [Integer] time until next change of the poll state
           #   (from paused to active)
@@ -162,12 +183,12 @@ module Karafka
             # If state is active, there is no date of change
             if state == 'active'
               %(
-                <span class="badge #{kafka_state_bg(state)} mt-1 mb-1">#{state}</span>
+                <span class="badge #{kafka_state_badge(state)}">#{state}</span>
               )
             elsif state_ch_in_seconds > year_in_seconds
               %(
                 <span
-                  class="badge #{kafka_state_bg(state)} mt-1 mb-1"
+                  class="badge #{kafka_state_badge(state)}"
                   title="until manual resume"
                 >
                   #{state}
@@ -176,7 +197,7 @@ module Karafka
             else
               %(
                 <span
-                  class="badge #{kafka_state_bg(state)} time-title mt-1 mb-1"
+                  class="badge #{kafka_state_badge(state)} time-title"
                   title="#{Time.now + state_ch_in_seconds}"
                 >
                   #{state}
@@ -191,7 +212,7 @@ module Karafka
           def lag_with_label(lag)
             if lag.negative?
               title = 'Not available until first offset commit'
-              %(<span class="badge bg-secondary" title="#{title}">N/A</span>)
+              %(<span class="badge badge-secondary" title="#{title}">N/A</span>)
             else
               lag.to_s
             end
@@ -207,7 +228,7 @@ module Karafka
           def offset_with_label(topic_name, partition_id, offset, explore: false)
             if offset.negative?
               title = 'Not available until first offset commit'
-              %(<span class="badge bg-secondary" title="#{title}">N/A</span>)
+              %(<span class="badge badge-secondary" title="#{title}">N/A</span>)
             elsif explore
               path = explorer_path(topic_name, partition_id, offset)
               %(<a href="#{path}">#{offset}</a>)
@@ -226,29 +247,34 @@ module Karafka
             when :at_risk
               'bg-warning bg-opacity-25'
             when :stopped
-              'bg-danger bg-opacity-25'
+              'bg-error bg-opacity-25'
+            else
+              raise ::Karafka::Errors::UnsupportedCaseError
+            end
+          end
+
+          # @param details [::Karafka::Web::Ui::Models::Partition] partition information with
+          #   lso risk state info
+          # @return [String] background classes for row marking
+          def lso_risk_state_badge(details)
+            case details.lso_risk_state
+            when :active
+              ''
+            when :at_risk
+              'badge-warning'
+            when :stopped
+              'badge-error'
             else
               raise ::Karafka::Errors::UnsupportedCaseError
             end
           end
 
-          # Returns the view title html code
+          # Sets the particular page title
           #
           # @param title [String] page title
-          # @param hr [Boolean] should we add the hr tag at the end
           # @return [String] title html
-          def view_title(title, hr: false)
-            <<-HTML
-              <div class="container mb-5">
-                <div class="row">
-                  <h3>
-                    #{title}
-                  </h3>
-                </div>
-
-                #{hr ? '<hr/>' : ''}
-              </div>
-            HTML
+          def view_title(title)
+            content_for(:title) { title }
           end
 
           # @param hash [Hash] we want to flatten
@@ -295,12 +321,12 @@ module Karafka
             path = current_path(sort: desc)
             full_name = "#{name}&nbsp;#{arrow_both}"
 
-            if params.sort == desc
+            if params.current_sort == desc
               path = current_path(sort: asc)
               full_name = "#{name}&nbsp;#{rev ? arrow_up : arrow_down}"
             end
 
-            if params.sort == asc
+            if params.current_sort == asc
               path = current_path(sort: desc)
               full_name = "#{name}&nbsp;#{rev ? arrow_down : arrow_up}"
             end
@@ -336,6 +362,13 @@ module Karafka
 
             %(<span title="#{string}">#{truncated}</span>)
           end
+
+          # Renders the svg icon out of our icon set
+          # @param name [String, Symbol] name of the icon
+          # @return [String] svg icon
+          def icon(name)
+            render "shared/icons/_#{name}"
+          end
         end
       end
     end

data/lib/karafka/web/ui/helpers/paths_helper.rb

@@ -6,6 +6,30 @@ module Karafka
       module Helpers
         # Helper for web ui paths builders
         module PathsHelper
+          # Helper method to flatten nested hashes and arrays
+          # @param prefix [String] The prefix for nested keys, initially an empty string.
+          # @param hash [Hash, Array] The nested hash or array to be flattened.
+          # @param [Hash] result The hash to store the flattened key-value pairs.
+          # @return [Hash] The flattened hash with keys in bracket notation suitable for URL
+          #   encoding.
+          def flatten_params(prefix, hash, result = {})
+            if hash.is_a?(Hash)
+              hash.each do |k, v|
+                new_prefix = prefix.empty? ? k.to_s : "#{prefix}[#{k}]"
+                flatten_params(new_prefix, v, result)
+              end
+            elsif hash.is_a?(Array)
+              hash.each_with_index do |v, i|
+                new_prefix = "#{prefix}[#{i}]"
+                flatten_params(new_prefix, v, result)
+              end
+            else
+              result[prefix] = hash.to_s
+            end
+
+            result
+          end
+
           # Generates a full path with the root path out of the provided arguments
           #
           # @param args [Array<String, Numeric>] arguments that will make the path

data/lib/karafka/web/ui/helpers/tailwind_helper.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Helpers
+        # Helper for generating tailwind rendering components with Ruby
+        # Simplifies many places in the UI
+        module TailwindHelper
+          # style types of components we support
+          TYPES = %i[
+            info
+            error
+            warning
+            success
+            primary
+            secondary
+          ].freeze
+
+          # @return [Array<Symbol>] style types of components we support
+          def tailwind_types
+            TYPES
+          end
+
+          # Renders a plain badge
+          # @param content [String] badge content
+          # @param classes [String] extra css classes
+          # @return [String] badge html
+          def badge(content, classes: '')
+            %(<span class="badge #{classes}">#{content}</span>)
+          end
+
+          # Renders a link to with button styling
+          # @param name [String] button name
+          # @param path [String] path to where to go
+          # @param classes [String] extra css classes
+          # @return [String] button link html
+          def link_button(name, path, classes: '')
+            %(<a href="#{path}" class="btn #{classes}">#{name}</a>)
+          end
+
+          # Defines various methods for badges and links that simplify defining them without
+          # having to provide whole classes scopes always.
+          TYPES.each do |type|
+            define_method :"badge_#{type}" do |content, classes: ''|
+              badge(content, classes: "#{classes} badge-#{type}")
+            end
+
+            define_method :"badge_#{type}_sm" do |content, classes: ''|
+              badge(content, classes: "#{classes} badge-#{type} badge-sm")
+            end
+
+            define_method :"link_button_#{type}" do |name, path, classes: ''|
+              link_button(name, path, classes: "#{classes} btn-#{type}")
+            end
+
+            define_method :"link_button_#{type}_sm" do |name, path, classes: ''|
+              link_button(name, path, classes: "#{classes} btn-#{type} btn-sm")
+            end
+
+            # @param message [String] alert message
+            # @return [String] html with alert
+            define_method :"alert_#{type}" do |message|
+              partial(
+                "shared/alerts/#{type}",
+                locals: {
+                  message: message
+                }
+              )
+            end
+
+            # @param message [String] alert message
+            # @return [String] html with alert
+            define_method :"alert_box_#{type}" do |title, description = nil, &block|
+              description = capture_erb(&block) if block
+
+              inject_erb partial(
+                "shared/alerts/box_#{type}",
+                locals: {
+                  title: title,
+                  description: description
+                }
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/lib/sorter.rb

@@ -33,7 +33,7 @@ module Karafka
             @field = '' unless allowed_attributes.include?(@field)
 
             # Things we have already seen and sorted. Prevents crashing on the circular
-            # dependencies sorting when same resources are present in different parts of the three
+            # dependencies sorting when same resources are present in different parts of the tree
             @seen = {}
           end
 

data/lib/karafka/web/ui/models/metrics/aggregated.rb

@@ -24,6 +24,7 @@ module Karafka
             #
             # If at least two elements do not exist for given delta range, we keep it empty
             DELTA_KEYS = %i[
+              jobs
               batches
               messages
               errors

data/lib/karafka/web/ui/models/metrics/charts/topics.rb

@@ -47,48 +47,64 @@ module Karafka
                 per_topic.merge('total sum' => total.to_a).to_json
               end
 
-              # @return [String] JSON with producers pace that represents high-watermarks sum for
-              #   each topic
-              def topics_pace
-                topics = {}
+              # @return [String] JSON with per-topic, highest LSO freeze duration. Useful for
+              #   debugging of issues arising from hanging transactions
+              def max_lso
+                topics = Hash.new { |h, k| h[k] = Hash.new { |h2, k2| h2[k2] = [] } }
 
                 @data.to_h.each do |topic, metrics|
                   topic_without_cg = topic.split('[').first
 
-                  # If we've already seen this topic data, we can skip
-                  next if topics.include?(topic_without_cg)
+                  metrics.each do |current|
+                    ls_offset_fd = current.last[:ls_offset_fd] || 0
 
-                  topics[topic_without_cg] = metrics.map do |current|
-                    [current.first, current.last[:pace]]
+                    # We convert this to seconds from milliseconds due to our Web UI precision
+                    # Reporting is in ms for consistency
+                    normalized_fd = (ls_offset_fd / 1_000.0).round
+
+                    topics[topic_without_cg][current.first] << normalized_fd
                   end
                 end
 
                 topics.each_value(&:compact!)
+                topics.each_value { |metrics| metrics.transform_values!(&:max) }
+                topics.transform_values! { |values| values.to_a.sort_by!(&:first) }
                 topics.to_json
               end
 
-              # @return [String] JSON with per-topic, highest LSO freeze duration. Useful for
-              #   debugging of issues arising from hanging transactions
-              def max_lso_time
-                topics = Hash.new { |h, k| h[k] = Hash.new { |h2, k2| h2[k2] = [] } }
+              # @return [String] JSON with producers pace that represents high-watermarks sum for
+              #   each topic.
+              #
+              # @note There is a case where data reported (sum on a topic) is lower then the
+              #   previous value. This can happen around rebalances because consumer may not
+              #   have all watermark offsets reported. This may cause consumers not to report some
+              #   of the partitions, effectively lowering the sum. Since high-watermark offsets can
+              #   only move forward, we compensate this by assuming that a lower value than
+              #   previous is an artefact of that type and we replace it with the max value we had
+              #   effectively compensating for under-reporting
+              def pace
+                topics = {}
 
                 @data.to_h.each do |topic, metrics|
                   topic_without_cg = topic.split('[').first
 
-                  metrics.each do |current|
-                    ls_offset_fd = current.last[:ls_offset_fd] || 0
+                  # If we've already seen this topic data, we can skip
+                  next if topics.include?(topic_without_cg)
 
-                    # We convert this to seconds from milliseconds due to our Web UI precision
-                    # Reporting is in ms for consistency
-                    normalized_fd = (ls_offset_fd / 1_000.0).round
+                  max_pace = 0
 
-                    topics[topic_without_cg][current.first] << normalized_fd
+                  topics[topic_without_cg] = metrics.map do |current|
+                    # Pace may be empty when for a given moment in time we got no info on
+                    # one of the topics. In such case we can compensate with max or 0
+                    current_pace = current.last[:pace] || 0
+
+                    max_pace = current_pace if current_pace > max_pace
+
+                    [current.first, max_pace]
                   end
                 end
 
                 topics.each_value(&:compact!)
-                topics.each_value { |metrics| metrics.transform_values!(&:max) }
-                topics.transform_values! { |values| values.to_a.sort_by!(&:first) }
                 topics.to_json
               end
             end