karafka-web 0.6.3 → 0.7.0

data/lib/karafka/web/ui/controllers/cluster.rb

@@ -8,7 +8,8 @@ module Karafka
         class Cluster < Base
           # List cluster info data
           def index
-            @cluster_info = Karafka::Admin.cluster_info
+            # Make sure, that for the cluster view we always get the most recent cluster state
+            @cluster_info = Models::ClusterInfo.fetch(cached: false)
 
             partitions_total = []
 
@@ -18,11 +19,13 @@ module Karafka
               end
             end
 
-            @partitions, @next_page = Ui::Lib::PaginateArray.new.call(
+            @partitions, last_page = Ui::Lib::Paginations::Paginators::Arrays.call(
               partitions_total,
               @params.current_page
             )
 
+            paginate(@params.current_page, !last_page)
+
             respond
           end
 
@@ -32,10 +35,13 @@ module Karafka
           # @return [Array<Hash>] array with topics to be displayed sorted in an alphabetical
           #   order
           def displayable_topics(cluster_info)
-            cluster_info
-              .topics
-              .reject { |topic| topic[:topic_name] == '__consumer_offsets' }
-              .sort_by { |topic| topic[:topic_name] }
+            all = cluster_info
+                  .topics
+                  .sort_by { |topic| topic[:topic_name] }
+
+            return all if ::Karafka::Web.config.ui.show_internal_topics
+
+            all.reject { |topic| topic[:topic_name].start_with?('__') }
           end
         end
       end

data/lib/karafka/web/ui/controllers/consumers.rb

@@ -9,13 +9,15 @@ module Karafka
           # List page with consumers
           # @note For now we load all and paginate over the squashed data.
           def index
-            @current_state = Models::State.current!
+            @current_state = Models::ConsumersState.current!
             @counters = Models::Counters.new(@current_state)
-            @processes, @next_page = Lib::PaginateArray.new.call(
+            @processes, last_page = Ui::Lib::Paginations::Paginators::Arrays.call(
               Models::Processes.active(@current_state),
               @params.current_page
             )
 
+            paginate(@params.current_page, !last_page)
+
             respond
           end
         end

data/lib/karafka/web/ui/controllers/dashboard.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Controllers
+        # Main Karafka Pro Web-Ui dashboard controller
+        class Dashboard < Ui::Controllers::Base
+          # View with statistics dashboard details
+          def index
+            @current_state = Models::ConsumersState.current!
+            @counters = Models::Counters.new(@current_state)
+
+            current_metrics = Models::ConsumersMetrics.current!
+
+            # Build the charts data using the aggregated metrics
+            @aggregated = Models::Metrics::Aggregated.new(
+              current_metrics.to_h.fetch(:aggregated)
+            )
+
+            # Load only historicals for the selected range
+            @aggregated_charts = Models::Metrics::Charts::Aggregated.new(
+              @aggregated, :seconds
+            )
+
+            respond
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/controllers/errors.rb

@@ -10,13 +10,15 @@ module Karafka
         class Errors < Base
           # Lists first page of the errors
           def index
-            @previous_page, @error_messages, @next_page, = Models::Message.page(
-              errors_topic,
-              0,
-              @params.current_page
-            )
-
             @watermark_offsets = Ui::Models::WatermarkOffsets.find(errors_topic, 0)
+            previous_offset, @error_messages, next_offset, = current_page_data
+
+            paginate(
+              previous_offset,
+              @params.current_offset,
+              next_offset,
+              @error_messages.map(&:offset)
+            )
 
             respond
           end
@@ -34,6 +36,17 @@ module Karafka
 
           private
 
+          # @return [Array] Array with requested messages as well as pagination details and other
+          #   obtained metadata
+          def current_page_data
+            Models::Message.offset_page(
+              errors_topic,
+              0,
+              @params.current_offset,
+              @watermark_offsets
+            )
+          end
+
           # @return [String] errors topic
           def errors_topic
             ::Karafka::Web.config.topics.errors

data/lib/karafka/web/ui/controllers/jobs.rb

@@ -8,7 +8,7 @@ module Karafka
         class Jobs < Base
           # Lists jobs
           def index
-            current_state = Models::State.current!
+            current_state = Models::ConsumersState.current!
             processes = Models::Processes.active(current_state)
 
             # Aggregate jobs and inject the process info into them for better reporting
@@ -19,11 +19,13 @@ module Karafka
               end
             end
 
-            @jobs, @next_page = Ui::Lib::PaginateArray.new.call(
+            @jobs, last_page = Ui::Lib::Paginations::Paginators::Arrays.call(
               jobs_total,
               @params.current_page
             )
 
+            paginate(@params.current_page, !last_page)
+
             respond
           end
         end

data/lib/karafka/web/ui/controllers/requests/params.rb

@@ -8,6 +8,17 @@ module Karafka
         module Requests
           # Internal representation of params with sane sanitization
           class Params
+            # What ranges we support for charts
+            # Anything else will be rejected
+            ALLOWED_RANGES = %w[
+              seconds
+              minutes
+              hours
+              days
+            ].freeze
+
+            private_constant :ALLOWED_RANGES
+
             # @param request_params [Hash] raw hash with params
             def initialize(request_params)
               @request_params = request_params
@@ -22,6 +33,23 @@ module Karafka
                 page.positive? ? page : 1
               end
             end
+
+            # @return [String] Range type for charts we want to fetch
+            def current_range
+              candidate = @request_params.fetch('range', 'seconds')
+              candidate = ALLOWED_RANGES.first unless ALLOWED_RANGES.include?(candidate)
+              candidate.to_sym
+            end
+
+            # @return [Integer] offset from which we want to start. `-1` indicates, that we want
+            #   to show the first page discovered based on the high watermark offset. If no offset
+            #   is provided, we go with the high offset first page approach
+            def current_offset
+              @current_offset ||= begin
+                offset = @request_params.fetch('offset', -1).to_i
+                offset < -1 ? -1 : offset
+              end
+            end
           end
         end
       end

data/lib/karafka/web/ui/controllers/responses/redirect.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Controllers
+        module Responses
+          # Representation of a redirect response with optional flash messages
+          class Redirect
+            attr_reader :path, :flashes
+
+            # @param path [String, Symbol] relative (without root path) path where we want to be
+            #   redirected or `:back` to use referer back
+            # @param flashes [Hash] hash where key is the flash type and value is the message
+            def initialize(path = :back, flashes = {})
+              @path = path
+              @flashes = flashes
+            end
+
+            # @return [Boolean] are we going back via referer and not explicit path
+            def back?
+              @path == :back
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/helpers/application_helper.rb

@@ -8,18 +8,6 @@ module Karafka
       module Helpers
         # Main application helper
         module ApplicationHelper
-          # Generates a full path with the root path out of the provided arguments
-          #
-          # @param args [Array<String, Numeric>] arguments that will make the path
-          # @return [String] path from the root
-          #
-          # @note This needs to be done that way with the `#root_path` because the web UI can be
-          #   mounted in a sub-path and we need to make sure our all paths are relative to "our"
-          #   root, not the root of the app in which it was mounted.
-          def root_path(*args)
-            "#{env.fetch('SCRIPT_NAME')}/#{args.join('/')}"
-          end
-
           # Adds active class to the current location in the nav if needed
           # @param location [Hash]
           def nav_class(location)
@@ -41,7 +29,7 @@ module Karafka
 
           # Renders per scope breadcrumbs
           def render_breadcrumbs
-            scope = request.path.gsub(root_path, '').split('/')[0]
+            scope = request.path.delete_prefix(root_path).split('/')[0]
 
             render "#{scope}/_breadcrumbs"
           end
@@ -137,19 +125,53 @@ module Karafka
             %(<span title="#{stamp}">#{time}</span>)
           end
 
+          # @param lag [Integer] lag
+          # @return [String] lag if correct or `N/A` with labeled explanation
+          # @see #offset_with_label
+          def lag_with_label(lag)
+            if lag.negative?
+              title = 'Not available until first offset commit'
+              %(<span class="badge bg-secondary" title="#{title}">N/A</span>)
+            else
+              lag.to_s
+            end
+          end
+
+          # @param topic_name [String] name of the topic for explorer path
+          # @param partition_id [Integer] partition for the explorer path
           # @param offset [Integer] offset
+          # @param explore [Boolean] should we generate (when allowed) a link to message explorer
           # @return [String] offset if correct or `N/A` with labeled explanation for offsets
           #   that are less than 0. Offset with less than 0 indicates, that the offset was not
           #   yet committed and there is no value we know of
-          def offset_with_label(offset)
+          def offset_with_label(topic_name, partition_id, offset, explore: false)
             if offset.negative?
               title = 'Not available until first offset commit'
               %(<span class="badge bg-secondary" title="#{title}">N/A</span>)
+            elsif explore
+              path = explorer_path(topic_name, partition_id, offset)
+              %(<a href="#{path}">#{offset}</a>)
             else
               offset.to_s
             end
           end
 
+          # @param details [::Karafka::Web::Ui::Models::Partition] partition information with
+          #   lso risk state info
+          # @return [String] background classes for row marking
+          def lso_risk_state_bg(details)
+            case details.lso_risk_state
+            when :active
+              ''
+            when :at_risk
+              'bg-warning bg-opacity-25'
+            when :stopped
+              'bg-danger bg-opacity-25'
+            else
+              raise ::Karafka::Errors::UnsupportedCaseError
+            end
+          end
+
           # Returns the view title html code
           #
           # @param title [String] page title
@@ -168,6 +190,27 @@ module Karafka
               </div>
             HTML
           end
+
+          # @param hash [Hash] we want to flatten
+          # @param parent_key [String] key for recursion
+          # @param result [Hash] result for recursion
+          # @return [Hash]
+          def flat_hash(hash, parent_key = nil, result = {})
+            hash.each do |key, value|
+              current_key = parent_key ? "#{parent_key}.#{key}" : key.to_s
+              if value.is_a?(Hash)
+                flat_hash(value, current_key, result)
+              elsif value.is_a?(Array)
+                value.each_with_index do |item, index|
+                  flat_hash({ index => item }, current_key, result)
+                end
+              else
+                result[current_key] = value
+              end
+            end
+
+            result
+          end
         end
       end
     end

data/lib/karafka/web/ui/helpers/paths_helper.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Helpers
+        # Helper for web ui paths builders
+        module PathsHelper
+          # Generates a full path with the root path out of the provided arguments
+          #
+          # @param args [Array<String, Numeric>] arguments that will make the path
+          # @return [String] path from the root
+          #
+          # @note This needs to be done that way with the `#root_path` because the web UI can be
+          #   mounted in a sub-path and we need to make sure our all paths are relative to "our"
+          #   root, not the root of the app in which it was mounted.
+          def root_path(*args)
+            "#{env.fetch('SCRIPT_NAME')}/#{args.join('/')}"
+          end
+
+          # Generates a full path to any asset with our web-ui version. We ship all assets with
+          # the version in the url to prevent those assets from being used after update. After
+          # each web-ui update, assets are going to be re-fetched as the url will change
+          #
+          # @param local_path [String] local path to the asset
+          # @return [String] full path to the asst including correct root path
+          def asset_path(local_path)
+            root_path("assets/#{Karafka::Web::VERSION}/#{local_path}")
+          end
+
+          # Helps build explorer paths. We often link offsets to proper messages, etc so this
+          # allows us to short-track this
+          # @param topic_name [String, nil] name of the topic where we want to go within the
+          #   explorer or nil if we want to just go to the explorer root
+          # @param partition_id [Integer, nil] partition we want to display in the explorer or nil
+          #   if we want to go to the topic root
+          # @param offset [Integer, nil] offset of particular message or nil of we want to just go
+          #   to the partition root
+          # @param action [String, nil] specific routed action or nil
+          # @return [String] path to the expected location
+          def explorer_path(topic_name = nil, partition_id = nil, offset = nil, action = nil)
+            root_path(*['explorer', topic_name, partition_id, offset, action].compact)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/lib/hash_proxy.rb

@@ -12,16 +12,17 @@ module Karafka
         # It is mostly used for flat hashes.
         #
         # It is in a way similar to openstruct but has abilities to dive deep into objects
+        #
+        # It is not super fast but it is enough for the UI and how deep structures we have.
         class HashProxy
+          extend Forwardable
+
+          def_delegators :@hash, :[], :[]=, :key?, :each, :find
+
           # @param hash [Hash] hash we want to convert to a proxy
           def initialize(hash)
             @hash = hash
-          end
-
-          # @param key [Object] hash key
-          # @return [Object] key content or nil if missing
-          def [](key)
-            @hash[key]
+            @visited = []
           end
 
           # @return [Original hash]
@@ -35,7 +36,12 @@ module Karafka
           def method_missing(method_name, *args, &block)
             return super unless args.empty? && block.nil?
 
+            @visited.clear
+
             result = deep_find(@hash, method_name.to_sym)
+
+            @visited.clear
+
             result.nil? ? super : result
           end
 
@@ -51,6 +57,12 @@ module Karafka
           # @param obj [Object] local scope of iterating
           # @param key [Symbol, String] key we are looking for
           def deep_find(obj, key)
+            # Prevent circular dependency lookups by making sure we do not check the same object
+            # multiple times
+            return nil if @visited.include?(obj)
+
+            @visited << obj
+
             if obj.respond_to?(:key?) && obj.key?(key)
               obj[key]
             elsif obj.respond_to?(:each)

data/lib/karafka/web/ui/lib/paginations/base.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Lib
+        # Namespace for all the types of pagination engines we want to support
+        module Paginations
+          # Abstraction on top of pagination, so we can alter pagination key and other things
+          # for non-standard pagination views (non page based, etc)
+          #
+          # @note We do not use `_page` explicitly to indicate, that the page scope may not operate
+          #   on numerable pages (1,2,3,4) but can operate on offsets or times, etc. `_offset` is
+          #   more general and may refer to many types of pagination.
+          class Base
+            attr_reader :previous_offset, :current_offset, :next_offset
+
+            # @return [Boolean] Should we show pagination at all
+            def paginate?
+              raise NotImplementedError, 'Implement in a subclass'
+            end
+
+            # @return [Boolean] Should first offset link be active. If false, the first offset link
+            #   will be disabled
+            def first_offset?
+              raise NotImplementedError, 'Implement in a subclass'
+            end
+
+            # @return [String] first offset url value
+            def first_offset
+              raise NotImplementedError, 'Implement in a subclass'
+            end
+
+            # @return [Boolean] Should previous offset link be active. If false, the previous
+            #   offset link will be disabled
+            def previous_offset?
+              raise NotImplementedError, 'Implement in a subclass'
+            end
+
+            # @return [Boolean] Should we show current offset. If false, the current offset link
+            #   will not be visible at all. Useful for non-linear pagination.
+            def current_offset?
+              raise NotImplementedError, 'Implement in a subclass'
+            end
+
+            # @return [Boolean] Should we show next offset pagination. If false, next offset link
+            #   will be disabled.
+            def next_offset?
+              raise NotImplementedError, 'Implement in a subclass'
+            end
+
+            # @return [String] the url offset key
+            def offset_key
+              raise NotImplementedError, 'Implement in a subclass'
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/lib/paginations/offset_based.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Lib
+        module Paginations
+          # Kafka offset based pagination backend
+          #
+          # Allows us to support paginating over offsets
+          class OffsetBased < Base
+            # @param previous_offset [Integer, false] previous offset or false if should not be
+            #   presented
+            # @param current_offset [Integer] current offset
+            # @param next_offset [Integer, Boolean] should we show next offset page button. If
+            #   false it will not be presented.
+            # @param visible_offsets [Array<Integer>] offsets that are visible in the paginated
+            #   view. It is needed for the current page label
+            def initialize(
+              previous_offset,
+              current_offset,
+              next_offset,
+              visible_offsets
+            )
+              @previous_offset = previous_offset
+              @current_offset = current_offset
+              @next_offset = next_offset
+              @visible_offsets = visible_offsets
+              super()
+            end
+
+            # Show pagination only when there is more than one page of results to be presented
+            #
+            # @return [Boolean]
+            def paginate?
+              @current_offset && (!!@previous_offset || !!@next_offset)
+            end
+
+            # @return [Boolean] active only when we are not on the first page. First page is always
+            #   indicated by the current offset being -1. If there is someone that sets up the
+            #   current offset to a value equal to the last message in the topic partition, we do
+            #   not consider it as a first page and we allow to "reset" to -1 via the first page
+            #   button
+            def first_offset?
+              @current_offset != -1
+            end
+
+            # @return [Boolean] first page offset is always nothing because we use the default -1
+            #   for the offset.
+            def first_offset
+              false
+            end
+
+            # @return [Boolean] Active previous page link when it is not the first page
+            def previous_offset?
+              !!@previous_offset
+            end
+
+            # @return [Boolean] We show current label with offsets that are present on the given
+            #   page
+            def current_offset?
+              true
+            end
+
+            # @return [Boolean] move to the next page if not false. False indicates, that there is
+            #   no next page to move to
+            def next_offset?
+              !!@next_offset
+            end
+
+            # If there is no next offset, we point to 0 as there should be no smaller offset than
+            # that in Kafka ever
+            # @return [Integer]
+            def next_offset
+              next_offset? ? @next_offset : 0
+            end
+
+            # @return [String] label of the current page. It is combined out of the first and
+            #   the last offsets to show the range where we are. It will be empty if no offsets
+            #   but this is not a problem as then we should not display pagination at all
+            def current_label
+              first = @visible_offsets.first
+              last = @visible_offsets.last
+              [first, last].compact.uniq.join(' - ').to_s
+            end
+
+            # @return [String] for offset based pagination we use the offset param name
+            def offset_key
+              'offset'
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/lib/paginations/page_based.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Lib
+        module Paginations
+          # Regular page-based pagination engine
+          class PageBased < Base
+            # @param current_offset [Integer] current page
+            # @param show_next_offset [Boolean] should we show next page
+            #   (value is computed automatically)
+            def initialize(
+              current_offset,
+              show_next_offset
+            )
+              @previous_offset = current_offset - 1
+              @current_offset = current_offset
+              @next_offset = show_next_offset ? current_offset + 1 : false
+              super()
+            end
+
+            # Show pagination only when there is more than one page
+            # @return [Boolean]
+            def paginate?
+              @current_offset && (@current_offset > 1 || !!@next_offset)
+            end
+
+            # @return [Boolean] active the first page link when we are not on the first page
+            def first_offset?
+              @current_offset > 1
+            end
+
+            # @return [Boolean] first page for page based pagination is always empty as it moves us
+            #   to the initial page so we do not include any page info
+            def first_offset
+              false
+            end
+
+            # @return [Boolean] Active previous page link when it is not the first page
+            def previous_offset?
+              @current_offset > 1
+            end
+
+            # @return [Boolean] always show current offset pagination value
+            def current_offset?
+              true
+            end
+
+            # @return [String] label of the current page
+            def current_label
+              @current_offset.to_s
+            end
+
+            # @return [Boolean] move to the next page if not false. False indicates, that there is
+            #   no next page to move to
+            def next_offset?
+              @next_offset
+            end
+
+            # @return [String] for page pages pagination, always use page as the url value
+            def offset_key
+              'page'
+            end
+          end
+        end
+      end
+    end
+  end
+end