karafka-web 0.7.9 → 0.8.0.rc1

data/lib/karafka/web/ui/lib/sorter.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Lib
+        # Sorting engine for deep in-memory structures
+        # It supports hashes, arrays and hash proxies.
+        #
+        # @note It handles sorting in place by mutating appropriate resources and sub-components
+        class Sorter
+          # We can support only two order types
+          ALLOWED_ORDERS = %w[asc desc].freeze
+
+          # Max depth for nested sorting
+          MAX_DEPTH = 8
+
+          private_constant :ALLOWED_ORDERS, :MAX_DEPTH
+
+          # @param sort_query [String] query for sorting or empty string if no sorting needed
+          # @param allowed_attributes [Array<String>] attributes on which we allow to sort. Since
+          #   we can sort on method invocations, this needs to be limited and provided on a per
+          #   controller basis.
+          def initialize(sort_query, allowed_attributes:)
+            field, order = sort_query.split(' ')
+
+            @order = order.to_s.downcase
+            @order = ALLOWED_ORDERS.first unless ALLOWED_ORDERS.include?(@order)
+
+            # Normalize the key since we do not operate on capitalized values
+            @field = field.to_s.downcase
+
+            @field = '' unless allowed_attributes.include?(@field)
+
+            # Things we have already seen and sorted. Prevents crashing on the circular
+            # dependencies sorting when same resources are present in different parts of the three
+            @seen = {}
+          end
+
+          # Sorts the structure and returns it sorted.
+          #
+          # @param resource [Hash, Array, Lib::HashProxy] structure we want to sort
+          # @param current_depth []
+          def call(resource, current_depth = 0)
+            # Skip if there is no sort field at all
+            return resource if @field.empty?
+            # Skip if we've already seen this resource
+            # We use object id instead of full object as the objects can get big
+            return resource if @seen.key?(resource.object_id)
+            # Skip if we are too deep
+            return resource if current_depth > MAX_DEPTH
+
+            @seen[resource.object_id] = nil
+
+            case resource
+            when Array
+              sort_array!(resource, current_depth)
+            when Hash
+              sort_hash!(resource, current_depth)
+            when Lib::HashProxy
+              # We can short hash in place here, because it will be still references (the same)
+              # in the hash proxy object, so we can do it that way
+              sort_hash!(resource.to_h, current_depth)
+            when Enumerable
+              sort_array!(resource, current_depth)
+            end
+
+            resource
+          end
+
+          private
+
+          # Sorts the hash in place
+          #
+          # @param hash [Hash] hash we want to sort
+          # @param current_depth [Integer] current depth of sorting from root
+          def sort_hash!(hash, current_depth)
+            # Run sorting on each value, since we may have nested hashes and arrays
+            hash.each do |key, value|
+              previous_key = @parent_key
+              @parent_key = key.to_s.downcase
+              call(value, current_depth + 1)
+              @parent_key = previous_key
+            end
+
+            # We cannot short hashes that are not type aligned. That is, we cannot compare
+            # nested hashes with integers, etc. In some cases we could (Float vs Integer), however
+            # for the same of simplicity, we do not to that
+            return unless hash.values.map(&:class).uniq.size == 1
+
+            # Allows sorting based on parent key when hash contains another hash where we want to
+            # sort based on the keys and not based on the value
+            if @parent_key == @field
+              # We also should not modify hashes that do not have values that are sortable
+              # false is sortable but nil is not
+              sorted = hash.sort_by { |key, _| key.to_s }
+            else
+              values = hash.values.map { |value| sortable_value(value) }
+
+              return if values.any?(&:nil?)
+              return unless values.map(&:class).uniq.size == 1
+
+              # Generate new hash that will have things in our desired order
+              sorted = hash.sort_by { |_, value| sortable_value(value) }
+            end
+
+            sorted.reverse! if desc?
+
+            # Clear our hash and inject the new values in the order in which we want to have them
+            # Such clear and merge will ensure things are in the order we desired them
+            hash.clear
+            hash.merge!(sorted.to_h)
+          end
+
+          # Sorts an array in-place based on a specified attribute.
+          #
+          # The method iterates over each element in the array and applies the transformation.
+          #
+          # @param array [Array<Object>] The array of elements to be sorted
+          # @param current_depth [Integer] The current depth of the sorting operation,
+          #   used in the `call` method to handle nested structures or recursion.
+          # @note This method modifies the array in place (mutates the caller).
+          def sort_array!(array, current_depth)
+            # Sort arrays containing hashes by a specific attribute
+            array.map! { |element| call(element, current_depth + 1) }
+
+            values = array.map { |element| sortable_value(element) }
+
+            return if values.any?(&:nil?)
+            return unless values.map(&:class).uniq.size == 1
+
+            array.sort_by! { |element| sortable_value(element) }
+            array.reverse! if desc?
+          end
+
+          # @return [Boolean] true if we sort in desc, otherwise false
+          def desc?
+            @order == 'desc'
+          end
+
+          # Extracts the attribute based on which we should sort (if present)
+          #
+          # @param element [Object] takes the element object and depending on its type, tries to
+          #   figure out the value based on which we may sort
+          # @return [Object, nil] sortable value or nil if nothing to sort
+          def sortable_value(element)
+            result = nil
+            result = element[@field] || element[@field.to_sym] if element.is_a?(Hash)
+            result = element.public_send(@field) if element.respond_to?(@field)
+
+            # We cannot sort on some of the types and some require mapping, thus we convert
+            # types here when needed
+            case result
+            when Hash
+              nil
+            when Lib::HashProxy
+              nil
+            when true
+              1
+            when false
+              0
+            else
+              result
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/models/counters.rb

@@ -17,6 +17,12 @@ module Karafka
             @hash[:errors] = estimate_errors_count
           end
 
+          # @return [Integer] number of jobs that are not yet running. This includes jobs on the
+          #   workers queue as well as jobs in the scheduling
+          def pending
+            enqueued + waiting
+          end
+
           private
 
           # Estimates the number of errors present in the errors topic.

data/lib/karafka/web/ui/models/health.rb

@@ -15,7 +15,7 @@ module Karafka
               fetch_topics_data(state, stats)
               fetch_rebalance_ages(state, stats)
 
-              stats
+              sort_structure(stats)
             end
 
             private
@@ -49,7 +49,7 @@ module Karafka
                   dispatched_at - rebalance_age_ms / 1_000
                 end
 
-                stats[cg_name][:rebalance_ages] ||= []
+                stats[cg_name][:rebalance_ages] ||= Set.new
                 stats[cg_name][:rebalance_ages] += ages
               end
 
@@ -80,6 +80,27 @@ module Karafka
                 end
               end
             end
+
+            # Sorts data so we always present it in an alphabetical order
+            #
+            # @param stats [Hash] stats hash
+            # @return [Hash] sorted data
+            def sort_structure(stats)
+              # Ensure that partitions for all topics are in correct order
+              # Ensure topics are in alphabetical order always
+              stats.each_value do |cg_data|
+                topics = cg_data[:topics]
+
+                topics.each do |topic_name, t_data|
+                  topics[topic_name] = Hash[t_data.sort_by { |key, _| key }]
+                end
+
+                cg_data[:topics] = Hash[topics.sort_by { |key, _| key }]
+              end
+
+              # Ensure that all consumer groups are always in the same order
+              Hash[stats.sort_by { |key, _| key }]
+            end
           end
         end
       end

data/lib/karafka/web/ui/models/jobs.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Models
+        # Model representing group of jobs
+        #
+        # It simplifies filtering on running jobs and others, etc
+        class Jobs
+          include Enumerable
+          extend Forwardable
+
+          # Last three methods are needed to provide sorting
+          def_delegators :@jobs_array, :empty?, :size, :map!, :sort_by!, :reverse!
+
+          # @param jobs_array [Array<Job>] all jobs we want to enclose
+          def initialize(jobs_array)
+            @jobs_array = jobs_array
+          end
+
+          # @return [Jobs] running jobs
+          def running
+            select { |job| job.status == 'running' }
+          end
+
+          # @return [Jobs] pending jobs
+          def pending
+            select { |job| job.status == 'pending' }
+          end
+
+          # Creates a new Jobs object with selected jobs
+          # @param block [Proc] select proc
+          # @return [Jobs] selected jobs enclosed with the Jobs object
+          def select(&block)
+            self.class.new(super(&block))
+          end
+
+          # Allows for iteration over jobs
+          # @param block [Proc] block to call for each job
+          def each(&block)
+            @jobs_array.each(&block)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/models/metrics/charts/aggregated.rb

@@ -12,9 +12,28 @@ module Karafka
               # @param aggregated [Hash] all aggregated for all periods
               # @param period [Symbol] period that we are interested in
               def initialize(aggregated, period)
+                @period = period
                 @data = aggregated.to_h.fetch(period)
               end
 
+              # @return [String] JSON with bytes sent and bytes received metrics
+              def data_transfers
+                scale_factor = Processing::TimeSeriesTracker::TIME_RANGES
+                               .fetch(@period)
+                               .fetch(:resolution)
+                               .then { |factor| factor / 1_024.to_f }
+
+                received = bytes_received.map do |element|
+                  [element[0], element[1] * scale_factor]
+                end
+
+                sent = bytes_sent.map do |element|
+                  [element[0], element[1] * scale_factor]
+                end
+
+                { received: received, sent: sent }.to_json
+              end
+
               # @param args [Array<String>] names of aggregated we want to show
               # @return [String] JSON with data about all the charts we were interested in
               def with(*args)
@@ -24,6 +43,20 @@ module Karafka
                   .to_json
               end
 
+              # @return [Array<Array<Symbol, Integer>>] active listeners statistics
+              def active_listeners
+                listeners.map do |listener|
+                  [listener[0], listener[1].fetch(:active)]
+                end
+              end
+
+              # @return [Array<Array<Symbol, Integer>>] standby listeners statistics
+              def standby_listeners
+                listeners.map do |listener|
+                  [listener[0], listener[1].fetch(:standby)]
+                end
+              end
+
               # @param method_name [String]
               # @param include_private [Boolean]
               def respond_to_missing?(method_name, include_private = false)

data/lib/karafka/web/ui/models/metrics/charts/topics.rb

@@ -76,20 +76,11 @@ module Karafka
                   topic_without_cg = topic.split('[').first
 
                   metrics.each do |current|
-                    ls_offset = current.last[:ls_offset] || 0
                     ls_offset_fd = current.last[:ls_offset_fd] || 0
-                    hi_offset = current.last[:hi_offset] || 0
 
                     # We convert this to seconds from milliseconds due to our Web UI precision
                     # Reporting is in ms for consistency
-                    normalized_fd = (ls_offset_fd / 1_000).round
-                    # In case ls_offset and hi_offset are the same, it means we're reached eof
-                    # and we just don't have more data. In cases like this, LSO freeze duration
-                    # will grow because LSO will remain unchanged, but it does not mean it is
-                    # frozen. It means there is just no more data in the topic partition
-                    # This means we need to nullify this case, otherwise it would report, that
-                    # lso is hanging.
-                    normalized_fd = 0 if ls_offset == hi_offset
+                    normalized_fd = (ls_offset_fd / 1_000.0).round
 
                     topics[topic_without_cg][current.first] << normalized_fd
                   end

data/lib/karafka/web/ui/models/process.rb

@@ -37,7 +37,8 @@ module Karafka
           def jobs
             super
               .map { |job| Job.new(job) }
-              .sort_by(&:started_at)
+              .sort_by(&:updated_at)
+              .then { |jobs| Jobs.new(jobs) }
           end
 
           # @return [Integer] collective stored lag on this process

data/lib/karafka/web/ui/models/status.rb

@@ -244,18 +244,34 @@ module Karafka
             )
           end
 
+          # @return [Status::Step] are there any active topics in the routing that are not present
+          #   in the cluster (does not apply to patterns)
+          def routing_topics_presence
+            if consumers_reports_schema_state.success?
+              existing = @cluster_info.topics.map { |topic| topic[:topic_name] }
+
+              missing = ::Karafka::App
+                        .routes
+                        .flat_map(&:topics)
+                        .flat_map { |topics| topics.map(&:itself) }
+                        .select(&:active?)
+                        .reject { |topic| topic.respond_to?(:patterns?) ? topic.patterns? : false }
+                        .map(&:name)
+                        .uniq
+                        .then { |routed_topics| routed_topics - existing }
+
+              Step.new(missing.empty? ? :success : :warning, missing)
+            else
+              Step.new(:halted, [])
+            end
+          end
+
           # @return [Status::Step] is Pro enabled with all of its features.
           # @note It's not an error not to have it but we want to warn, that some of the features
           #   may not work without Pro.
           def pro_subscription
-            status = if consumers_reports_schema_state.success?
-                       ::Karafka.pro? ? :success : :warning
-                     else
-                       :halted
-                     end
-
             Step.new(
-              status,
+              ::Karafka.pro? ? :success : :warning,
               nil
             )
           end

data/lib/karafka/web/ui/models/topic.rb

@@ -8,7 +8,9 @@ module Karafka
         class Topic < Lib::HashProxy
           # @return [Array<Partition>] All topic partitions data
           def partitions
-            super.values.map do |partition_hash|
+            super.map do |partition_id, partition_hash|
+              partition_hash[:partition_id] = partition_id
+
               Partition.new(partition_hash)
             end
           end

data/lib/karafka/web/ui/models/visibility_filter.rb

@@ -26,6 +26,22 @@ module Karafka
           def payload?(message)
             !message.headers.key?('encryption')
           end
+
+          # Should it be allowed to download this message raw payload
+          #
+          # @param message [::Karafka::Messages::Message]
+          # @return [Boolean] true if downloads allowed
+          def download?(message)
+            payload?(message)
+          end
+
+          # Should it be allowed to download the deserialized and sanitized payload as JSON
+          #
+          # @param message [::Karafka::Messages::Message]
+          # @return [Boolean] true if exports allowed
+          def export?(message)
+            payload?(message)
+          end
         end
       end
     end

data/lib/karafka/web/ui/pro/app.rb

@@ -49,8 +49,16 @@ module Karafka
             r.on 'consumers' do
               controller = Controllers::Consumers.new(params)
 
-              r.get String, 'jobs' do |process_id|
-                controller.jobs(process_id)
+              r.on String, 'jobs' do |process_id|
+                r.get 'running' do
+                  controller.running_jobs(process_id)
+                end
+
+                r.get 'pending' do
+                  controller.pending_jobs(process_id)
+                end
+
+                r.redirect root_path("consumers/#{process_id}/jobs/running")
               end
 
               r.get String, 'subscriptions' do |process_id|
@@ -67,9 +75,18 @@ module Karafka
               end
             end
 
-            r.get 'jobs' do
+            r.on 'jobs' do
               controller = Controllers::Jobs.new(params)
-              controller.index
+
+              r.get 'running' do
+                controller.running
+              end
+
+              r.get 'pending' do
+                controller.pending
+              end
+
+              r.redirect root_path('jobs/running')
             end
 
             r.on 'routing' do
@@ -133,6 +150,14 @@ module Karafka
               r.post String, Integer, Integer, 'republish' do |topic_id, partition_id, offset|
                 controller.republish(topic_id, partition_id, offset)
               end
+
+              r.get String, Integer, Integer, 'download' do |topic_id, partition_id, offset|
+                controller.download(topic_id, partition_id, offset)
+              end
+
+              r.get String, Integer, Integer, 'export' do |topic_id, partition_id, offset|
+                controller.export(topic_id, partition_id, offset)
+              end
             end
 
             r.on 'health' do
@@ -146,14 +171,27 @@ module Karafka
                 controller.overview
               end
 
+              r.get 'changes' do
+                controller.changes
+              end
+
               r.get do
                 r.redirect root_path('health/overview')
               end
             end
 
-            r.get 'cluster' do
+            r.on 'cluster' do
               controller = Controllers::Cluster.new(params)
-              controller.index
+
+              r.get 'brokers' do
+                controller.brokers
+              end
+
+              r.get 'topics' do
+                controller.topics
+              end
+
+              r.redirect root_path('cluster/brokers')
             end
 
             r.on 'errors' do

data/lib/karafka/web/ui/pro/controllers/cluster.rb

@@ -19,6 +19,7 @@ module Karafka
         module Controllers
           # Cluster controller
           class Cluster < Ui::Controllers::Cluster
+            self.sortable_attributes = Ui::Controllers::Cluster.sortable_attributes
           end
         end
       end

data/lib/karafka/web/ui/pro/controllers/consumers.rb

@@ -18,18 +18,39 @@ module Karafka
         module Controllers
           # Controller for displaying consumers states and details about them
           class Consumers < Ui::Controllers::Base
+            self.sortable_attributes = %w[
+              name
+              started_at
+              lag_stored
+              id
+              lag_stored_d
+              committed_offset
+              stored_offset
+              fetch_state
+              poll_state
+              lso_risk_state
+              topic
+              consumer
+              type
+              messages
+              first_offset
+              last_offset
+              updated_at
+            ].freeze
+
             # Consumers list
             def index
               @current_state = Models::ConsumersState.current!
               @counters = Models::Counters.new(@current_state)
+
               @processes, last_page = Lib::Paginations::Paginators::Arrays.call(
-                Models::Processes.active(@current_state),
+                refine(Models::Processes.active(@current_state)),
                 @params.current_page
               )
 
               paginate(@params.current_page, !last_page)
 
-              respond
+              render
             end
 
             # @param process_id [String] id of the process we're interested in
@@ -37,19 +58,44 @@ module Karafka
               current_state = Models::ConsumersState.current!
               @process = Models::Process.find(current_state, process_id)
 
-              respond
+              render
+            end
+
+            # Renders details about running jobs
+            #
+            # @param process_id [String] id of the process we're interested in
+            def running_jobs(process_id)
+              details(process_id)
+
+              @running_jobs = @process.jobs.running
+
+              refine(@running_jobs)
+
+              render
             end
 
+            # Renders details about pending jobs
+            #
             # @param process_id [String] id of the process we're interested in
-            def jobs(process_id)
+            def pending_jobs(process_id)
               details(process_id)
-              respond
+
+              @pending_jobs = @process.jobs.pending
+
+              refine(@pending_jobs)
+
+              render
             end
 
             # @param process_id [String] id of the process we're interested in
             def subscriptions(process_id)
               details(process_id)
-              respond
+
+              # We want to have sorting but on a per subscription group basis and not to sort
+              # everything
+              @process.consumer_groups.each { |subscription_group| refine(subscription_group) }
+
+              render
             end
           end
         end

data/lib/karafka/web/ui/pro/controllers/dashboard.rb

@@ -44,7 +44,7 @@ module Karafka
                 @topics, @params.current_range
               )
 
-              respond
+              render
             end
           end
         end

data/lib/karafka/web/ui/pro/controllers/dlq.rb

@@ -33,7 +33,7 @@ module Karafka
                             .select { |topic| dlq_topic_names.include?(topic[:topic_name]) }
                             .sort_by { |topic| topic[:topic_name] }
 
-              respond
+              render
             end
           end
         end

data/lib/karafka/web/ui/pro/controllers/errors.rb

@@ -35,7 +35,7 @@ module Karafka
 
               paginate(@params.current_page, next_page)
 
-              respond
+              render
             end
 
             # @param partition_id [Integer] id of the partition of errors we are interested in
@@ -58,7 +58,7 @@ module Karafka
                 @error_messages.map(&:offset)
               )
 
-              respond
+              render
             end
 
             # Shows given error details
@@ -77,7 +77,7 @@ module Karafka
               watermark_offsets = Ui::Models::WatermarkOffsets.find(errors_topic, partition_id)
               paginate(offset, watermark_offsets.low, watermark_offsets.high)
 
-              respond
+              render
             end
 
             private