karafka-web 0.6.2 → 0.7.0

data/lib/karafka/web/ui/controllers/jobs.rb

@@ -8,7 +8,7 @@ module Karafka
         class Jobs < Base
           # Lists jobs
           def index
-            current_state = Models::State.current!
+            current_state = Models::ConsumersState.current!
             processes = Models::Processes.active(current_state)
 
             # Aggregate jobs and inject the process info into them for better reporting

data/lib/karafka/web/ui/controllers/requests/params.rb

@@ -8,6 +8,17 @@ module Karafka
         module Requests
           # Internal representation of params with sane sanitization
           class Params
+            # What ranges we support for charts
+            # Anything else will be rejected
+            ALLOWED_RANGES = %w[
+              seconds
+              minutes
+              hours
+              days
+            ].freeze
+
+            private_constant :ALLOWED_RANGES
+
             # @param request_params [Hash] raw hash with params
             def initialize(request_params)
               @request_params = request_params
@@ -23,6 +34,13 @@ module Karafka
               end
             end
 
+            # @return [String] Range type for charts we want to fetch
+            def current_range
+              candidate = @request_params.fetch('range', 'seconds')
+              candidate = ALLOWED_RANGES.first unless ALLOWED_RANGES.include?(candidate)
+              candidate.to_sym
+            end
+
             # @return [Integer] offset from which we want to start. `-1` indicates, that we want
             #   to show the first page discovered based on the high watermark offset. If no offset
             #   is provided, we go with the high offset first page approach

data/lib/karafka/web/ui/controllers/responses/redirect.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Controllers
+        module Responses
+          # Representation of a redirect response with optional flash messages
+          class Redirect
+            attr_reader :path, :flashes
+
+            # @param path [String, Symbol] relative (without root path) path where we want to be
+            #   redirected or `:back` to use referer back
+            # @param flashes [Hash] hash where key is the flash type and value is the message
+            def initialize(path = :back, flashes = {})
+              @path = path
+              @flashes = flashes
+            end
+
+            # @return [Boolean] are we going back via referer and not explicit path
+            def back?
+              @path == :back
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/helpers/application_helper.rb

@@ -8,18 +8,6 @@ module Karafka
       module Helpers
         # Main application helper
         module ApplicationHelper
-          # Generates a full path with the root path out of the provided arguments
-          #
-          # @param args [Array<String, Numeric>] arguments that will make the path
-          # @return [String] path from the root
-          #
-          # @note This needs to be done that way with the `#root_path` because the web UI can be
-          #   mounted in a sub-path and we need to make sure our all paths are relative to "our"
-          #   root, not the root of the app in which it was mounted.
-          def root_path(*args)
-            "#{env.fetch('SCRIPT_NAME')}/#{args.join('/')}"
-          end
-
           # Adds active class to the current location in the nav if needed
           # @param location [Hash]
           def nav_class(location)
@@ -41,7 +29,7 @@ module Karafka
 
           # Renders per scope breadcrumbs
           def render_breadcrumbs
-            scope = request.path.gsub(root_path, '').split('/')[0]
+            scope = request.path.delete_prefix(root_path).split('/')[0]
 
             render "#{scope}/_breadcrumbs"
           end
@@ -137,19 +125,53 @@ module Karafka
             %(<span title="#{stamp}">#{time}</span>)
           end
 
+          # @param lag [Integer] lag
+          # @return [String] lag if correct or `N/A` with labeled explanation
+          # @see #offset_with_label
+          def lag_with_label(lag)
+            if lag.negative?
+              title = 'Not available until first offset commit'
+              %(<span class="badge bg-secondary" title="#{title}">N/A</span>)
+            else
+              lag.to_s
+            end
+          end
+
+          # @param topic_name [String] name of the topic for explorer path
+          # @param partition_id [Integer] partition for the explorer path
           # @param offset [Integer] offset
+          # @param explore [Boolean] should we generate (when allowed) a link to message explorer
           # @return [String] offset if correct or `N/A` with labeled explanation for offsets
           #   that are less than 0. Offset with less than 0 indicates, that the offset was not
           #   yet committed and there is no value we know of
-          def offset_with_label(offset)
+          def offset_with_label(topic_name, partition_id, offset, explore: false)
             if offset.negative?
               title = 'Not available until first offset commit'
               %(<span class="badge bg-secondary" title="#{title}">N/A</span>)
+            elsif explore
+              path = explorer_path(topic_name, partition_id, offset)
+              %(<a href="#{path}">#{offset}</a>)
             else
               offset.to_s
             end
           end
 
+          # @param details [::Karafka::Web::Ui::Models::Partition] partition information with
+          #   lso risk state info
+          # @return [String] background classes for row marking
+          def lso_risk_state_bg(details)
+            case details.lso_risk_state
+            when :active
+              ''
+            when :at_risk
+              'bg-warning bg-opacity-25'
+            when :stopped
+              'bg-danger bg-opacity-25'
+            else
+              raise ::Karafka::Errors::UnsupportedCaseError
+            end
+          end
+
           # Returns the view title html code
           #
           # @param title [String] page title
@@ -168,6 +190,27 @@ module Karafka
               </div>
             HTML
           end
+
+          # @param hash [Hash] we want to flatten
+          # @param parent_key [String] key for recursion
+          # @param result [Hash] result for recursion
+          # @return [Hash]
+          def flat_hash(hash, parent_key = nil, result = {})
+            hash.each do |key, value|
+              current_key = parent_key ? "#{parent_key}.#{key}" : key.to_s
+              if value.is_a?(Hash)
+                flat_hash(value, current_key, result)
+              elsif value.is_a?(Array)
+                value.each_with_index do |item, index|
+                  flat_hash({ index => item }, current_key, result)
+                end
+              else
+                result[current_key] = value
+              end
+            end
+
+            result
+          end
         end
       end
     end

data/lib/karafka/web/ui/helpers/paths_helper.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Helpers
+        # Helper for web ui paths builders
+        module PathsHelper
+          # Generates a full path with the root path out of the provided arguments
+          #
+          # @param args [Array<String, Numeric>] arguments that will make the path
+          # @return [String] path from the root
+          #
+          # @note This needs to be done that way with the `#root_path` because the web UI can be
+          #   mounted in a sub-path and we need to make sure our all paths are relative to "our"
+          #   root, not the root of the app in which it was mounted.
+          def root_path(*args)
+            "#{env.fetch('SCRIPT_NAME')}/#{args.join('/')}"
+          end
+
+          # Generates a full path to any asset with our web-ui version. We ship all assets with
+          # the version in the url to prevent those assets from being used after update. After
+          # each web-ui update, assets are going to be re-fetched as the url will change
+          #
+          # @param local_path [String] local path to the asset
+          # @return [String] full path to the asst including correct root path
+          def asset_path(local_path)
+            root_path("assets/#{Karafka::Web::VERSION}/#{local_path}")
+          end
+
+          # Helps build explorer paths. We often link offsets to proper messages, etc so this
+          # allows us to short-track this
+          # @param topic_name [String, nil] name of the topic where we want to go within the
+          #   explorer or nil if we want to just go to the explorer root
+          # @param partition_id [Integer, nil] partition we want to display in the explorer or nil
+          #   if we want to go to the topic root
+          # @param offset [Integer, nil] offset of particular message or nil of we want to just go
+          #   to the partition root
+          # @param action [String, nil] specific routed action or nil
+          # @return [String] path to the expected location
+          def explorer_path(topic_name = nil, partition_id = nil, offset = nil, action = nil)
+            root_path(*['explorer', topic_name, partition_id, offset, action].compact)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/lib/hash_proxy.rb

@@ -12,16 +12,17 @@ module Karafka
         # It is mostly used for flat hashes.
         #
         # It is in a way similar to openstruct but has abilities to dive deep into objects
+        #
+        # It is not super fast but it is enough for the UI and how deep structures we have.
         class HashProxy
+          extend Forwardable
+
+          def_delegators :@hash, :[], :[]=, :key?, :each, :find
+
           # @param hash [Hash] hash we want to convert to a proxy
           def initialize(hash)
             @hash = hash
-          end
-
-          # @param key [Object] hash key
-          # @return [Object] key content or nil if missing
-          def [](key)
-            @hash[key]
+            @visited = []
           end
 
           # @return [Original hash]
@@ -35,7 +36,12 @@ module Karafka
           def method_missing(method_name, *args, &block)
             return super unless args.empty? && block.nil?
 
+            @visited.clear
+
             result = deep_find(@hash, method_name.to_sym)
+
+            @visited.clear
+
             result.nil? ? super : result
           end
 
@@ -51,6 +57,12 @@ module Karafka
           # @param obj [Object] local scope of iterating
           # @param key [Symbol, String] key we are looking for
           def deep_find(obj, key)
+            # Prevent circular dependency lookups by making sure we do not check the same object
+            # multiple times
+            return nil if @visited.include?(obj)
+
+            @visited << obj
+
             if obj.respond_to?(:key?) && obj.key?(key)
               obj[key]
             elsif obj.respond_to?(:each)

data/lib/karafka/web/ui/lib/paginations/watermark_offsets_based.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Lib
+        module Paginations
+          # Watermark offsets single message pagination engine
+          #
+          # It is used to provide pagination for single message displays (explorer, errors)
+          class WatermarkOffsetsBased < Base
+            # @param current_offset [Integer] current message offset
+            # @param low_watermark_offset [Integer]
+            # @param high_watermark_offset [Integer]
+            def initialize(
+              current_offset,
+              low_watermark_offset,
+              high_watermark_offset
+            )
+              @low_watermark_offset = low_watermark_offset
+              @high_watermark_offset = high_watermark_offset
+              @previous_offset = current_offset + 1
+              @current_offset = current_offset
+              @next_offset = current_offset - 1
+              super()
+            end
+
+            # @return [Boolean] show pagination only when there are other things to present
+            def paginate?
+              return true if @current_offset > @low_watermark_offset
+              return true if @current_offset < @high_watermark_offset - 1
+
+              false
+            end
+
+            # @return [Boolean] provide link to the first (newest)
+            def first_offset?
+              @current_offset < @high_watermark_offset - 1
+            end
+
+            # @return [Integer] highest available offset
+            def first_offset
+              @high_watermark_offset - 1
+            end
+
+            # @return [Boolean]
+            def previous_offset?
+              @current_offset < @high_watermark_offset - 1
+            end
+
+            # @return [Boolean] We always show current offset
+            def current_offset?
+              true
+            end
+
+            # @return [String] shows as current page pagination the offset
+            def current_label
+              @current_offset.to_s
+            end
+
+            # @return [Boolean] if not lowest, show
+            def next_offset?
+              @current_offset > @low_watermark_offset
+            end
+
+            # @return [String] params offset key
+            def offset_key
+              'offset'
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/lib/ttl_cache.rb

@@ -56,6 +56,14 @@ module Karafka
             end
           end
 
+          # Clears the whole cache
+          def clear
+            @mutex.synchronize do
+              @times.clear
+              @values.clear
+            end
+          end
+
           private
 
           # Removes expired elements from the cache

data/lib/karafka/web/ui/models/consumers_metrics.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Models
+        # Model representing the current consumers metrics most recent state
+        class ConsumersMetrics < Lib::HashProxy
+          class << self
+            # @return [State, false] current consumers metrics or false if not found
+            def current
+              state = fetch
+
+              return false unless state
+
+              # Do not return the state in case web-ui is not enabled because we need our
+              # internal deserializer for it to operate. False will force user to go to the
+              # status page
+              return false unless Models::Status.new.enabled.success?
+
+              state = state.payload
+              new(state)
+            end
+
+            # @return [State] current consumers metrics
+            # @raise [::Karafka::Web::Errors::Ui::NotFoundError] raised when there is no metrics.
+            def current!
+              current || raise(::Karafka::Web::Errors::Ui::NotFoundError)
+            end
+
+            private
+
+            # @return [::Karafka::Messages::Message, nil] most recent state or nil if none
+            def fetch
+              ::Karafka::Admin.read_topic(
+                Karafka::Web.config.topics.consumers.metrics,
+                0,
+                1
+              ).last
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/models/{state.rb → consumers_state.rb}

@@ -7,7 +7,7 @@ module Karafka
         # Represents the current consumer processes aggregated state
         # This state is the core of Karafka reporting. It holds the most important aggregated data
         # as well as pointers to states of particular consumers and their details.
-        class State < Lib::HashProxy
+        class ConsumersState < Lib::HashProxy
           extend ::Karafka::Core::Helpers::Time
 
           class << self
@@ -18,6 +18,10 @@ module Karafka
               state = fetch
 
               return false unless state
+              # Do not return the state in case web-ui is not enabled because we need our
+              # internal deserializer for it to operate. False will force user to go to the
+              # status page
+              return false unless Models::Status.new.enabled.success?
 
               state = state.payload
               evict_expired_processes(state)
@@ -28,7 +32,7 @@ module Karafka
 
             # @return [State] current aggregated state
             # @raise [::Karafka::Web::Errors::Ui::NotFoundError] raised when there is no current
-            #   state. Probably because `karafka server` was never executed
+            #   state.
             def current!
               current || raise(::Karafka::Web::Errors::Ui::NotFoundError)
             end

data/lib/karafka/web/ui/models/health.rb

@@ -12,21 +12,51 @@ module Karafka
             def current(state)
               stats = {}
 
+              fetch_topics_data(state, stats)
+              fetch_rebalance_ages(state, stats)
+
+              stats
+            end
+
+            private
+
+            # Aggregates data on a per topic basis (in the context of a consumer group)
+            # @param state [Hash]
+            # @param stats [Hash] hash where we will store all the aggregated data
+            def fetch_topics_data(state, stats)
               iterate_partitions(state) do |process, consumer_group, topic, partition|
                 cg_name = consumer_group.id
                 t_name = topic.name
                 pt_id = partition.id
 
-                stats[cg_name] ||= {}
-                stats[cg_name][t_name] ||= {}
-                stats[cg_name][t_name][pt_id] = partition.to_h
-                stats[cg_name][t_name][pt_id][:process] = process
+                stats[cg_name] ||= { topics: {} }
+                stats[cg_name][:topics][t_name] ||= {}
+                stats[cg_name][:topics][t_name][pt_id] = partition
+                stats[cg_name][:topics][t_name][pt_id][:process] = process
               end
-
-              stats
             end
 
-            private
+            # Aggregates rebalances ages data
+            # @param state [Hash]
+            # @param stats [Hash] hash where we will store all the aggregated data
+            def fetch_rebalance_ages(state, stats)
+              iterate_partitions(state) do |process, consumer_group|
+                cg_name = consumer_group.id
+                dispatched_at = process.dispatched_at
+
+                ages = consumer_group[:subscription_groups].values.map do |sub_group_details|
+                  rebalance_age_ms = sub_group_details[:state][:rebalance_age] || 0
+                  dispatched_at - rebalance_age_ms / 1_000
+                end
+
+                stats[cg_name][:rebalance_ages] ||= []
+                stats[cg_name][:rebalance_ages] += ages
+              end
+
+              stats.each_value do |details|
+                details[:rebalanced_at] = details[:rebalance_ages].max
+              end
+            end
 
             # Iterates over all partitions, yielding with extra expanded details
             #

data/lib/karafka/web/ui/models/message.rb

@@ -104,7 +104,7 @@ module Karafka
                   # If there is a potential previous page with more recent data, compute its
                   # offset
                   previous_offset >= high_offset ? false : previous_offset,
-                  fill_compacted(messages, partition_id, context_offset, context_count).reverse,
+                  fill_compacted(messages, partition_id, context_offset, context_count, high_offset).reverse,
                   next_offset
                 ]
               end
@@ -215,14 +215,22 @@ module Karafka
             # @param start_offset [Integer] offset of the first message (lowest) that we received
             # @param count [Integer] how many messages we wanted - we need that to fill spots to
             #   have exactly the number that was  requested and not more
+            # @param high_offset [Integer] high watermark offset
             # @return [Array<Karafka::Messages::Message, Integer>] array with gaps filled with the
             #   missing offset
-            def fill_compacted(messages, partition_id, start_offset, count)
-              Array.new(count) do |index|
+            def fill_compacted(messages, partition_id, start_offset, count, high_offset)
+              filled = Array.new(count) do |index|
                 messages.find do |message|
                   (message.offset - start_offset) == index
                 end || [partition_id, start_offset + index]
               end
+
+              # Remove dummies provisioned over the high offset
+              filled.delete_if do |message|
+                message.is_a?(Array) && message.last >= high_offset
+              end
+
+              filled
             end
           end
         end