karafka-web 0.6.3 → 0.7.0

data/lib/karafka/web/ui/lib/paginations/paginators/arrays.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Lib
+        module Paginations
+          # Namespace for commands that build paginated resources based on the provided page
+          module Paginators
+            # A simple wrapper for paginating array related data structures
+            # We call this with plural (same with `Sets`) to avoid confusion with Ruby classes
+            class Arrays < Base
+              class << self
+                # @param array [Array] array we want to paginate
+                # @param current_page [Integer] page we want to be on
+                # @return [Array<Array, Boolean>] Array with two elements: first is the array with
+                #   data of the given page and second is a boolean flag with info if the elements we got
+                #   are from the last page
+                def call(array, current_page)
+                  slices = array.each_slice(per_page).to_a
+                  current_data = slices[current_page - 1] || []
+                  last_page = !(slices.count >= current_page - 1 && current_data.size >= per_page)
+
+                  [current_data, last_page]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/lib/paginations/paginators/base.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Lib
+        module Paginations
+          module Paginators
+            # Base paginator
+            class Base
+              class << self
+                # @return [Integer] number of elements per page
+                def per_page
+                  ::Karafka::Web.config.ui.per_page
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/lib/paginations/paginators/partitions.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Lib
+        module Paginations
+          module Paginators
+            # Paginator for selecting proper range of partitions for each page
+            # For topics with a lot of partitions we cannot get all the data efficiently, that
+            # is why we limit number of partitions per page and reduce the operations
+            # that way. This allows us to effectively display more while not having to fetch
+            # more partitions then the number of messages per page.
+            # In cases like this we distribute partitions evenly part of partitions on each of
+            # the pages. This may become unreliable for partitions that are not evenly
+            # distributed but this allows us to display data for as many partitions as we want
+            # without overloading the system
+            class Partitions < Base
+              class << self
+                # Computers the partitions slice, materialized page and the limitations status
+                # for a given page
+                # @param partitions_count [Integer] number of partitions for a given topic
+                # @param current_page [Integer] current page
+                # @return [Array<Array<Integer>, Integer, Boolean>] list of partitions that should
+                #   be active on a given page, materialized page for them and info if we had to
+                #   limit the partitions number on a given page
+                def call(partitions_count, current_page)
+                  # How many "chunks" of partitions we will have
+                  slices_count = (partitions_count / per_page.to_f).ceil
+                  # How many partitions in a single slice should we have
+                  in_slice = (partitions_count / slices_count.to_f).ceil
+                  # Which "chunked" page do we want to get
+                  materialized_page = (current_page / slices_count.to_f).ceil
+                  # Which slice is the one we are operating on
+                  active_slice_index = (current_page - 1) % slices_count
+                  # All available slices so we can pick one that is active
+                  partitions_slices = (0...partitions_count).each_slice(in_slice).to_a
+                  # Select active partitions only
+                  active_partitions = partitions_slices[active_slice_index]
+                  # Are we limiting ourselves because of partition count
+                  limited = slices_count > 1
+
+                  [active_partitions, materialized_page, limited]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/lib/paginations/paginators/sets.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Lib
+        module Paginations
+          module Paginators
+            # Paginator that allows us to take several lists/sets and iterate over them in a
+            # round-robin fashion.
+            #
+            # It does not have to iterate over all the elements from each set for higher pages
+            # making it much more effective than the naive implementation.
+            class Sets < Base
+              class << self
+                # @param counts [Array<Integer>] sets elements counts
+                # @param current_page [Integer] page number
+                # @return [Hash<Integer, Range>] hash with integer keys indicating the count
+                #   location and the range needed to be taken of elements (counting backwards) for
+                #   each partition
+                def call(counts, current_page)
+                  return {} if current_page < 1
+
+                  lists = counts.dup.map.with_index { |el, i| [i, el] }
+
+                  curr_item_index = 0
+                  curr_list_index = 0
+                  items_to_skip_count = per_page * (current_page - 1)
+
+                  loop do
+                    lists_count = lists.length
+                    return {} if lists_count.zero?
+
+                    shortest_list_count = lists.map(&:last).min
+                    mover = (shortest_list_count - curr_item_index)
+                    items_we_are_considering_count = lists_count * mover
+
+                    if items_we_are_considering_count >= items_to_skip_count
+                      curr_item_index += items_to_skip_count / lists_count
+                      curr_list_index = items_to_skip_count % lists_count
+                      break
+                    else
+                      curr_item_index = shortest_list_count
+                      lists.delete_if { |x| x.last == shortest_list_count }
+                      items_to_skip_count -= items_we_are_considering_count
+                    end
+                  end
+
+                  page_items = []
+                  largest_list_count = lists.map(&:last).max
+
+                  while page_items.length < per_page && curr_item_index < largest_list_count
+                    curr_list = lists[curr_list_index]
+
+                    if curr_item_index < curr_list.last
+                      page_items << [curr_list.first, curr_item_index]
+                    end
+
+                    curr_list_index += 1
+                    if curr_list_index == lists.length
+                      curr_list_index = 0
+                      curr_item_index += 1
+                    end
+                  end
+
+                  hashed = Hash.new { |h, k| h[k] = [] }
+
+                  page_items.each do |el|
+                    hashed[el.first] << el.last
+                  end
+
+                  hashed.each do |key, value|
+                    hashed[key] = (value.first..value.last)
+                  end
+
+                  hashed
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/lib/paginations/watermark_offsets_based.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Lib
+        module Paginations
+          # Watermark offsets single message pagination engine
+          #
+          # It is used to provide pagination for single message displays (explorer, errors)
+          class WatermarkOffsetsBased < Base
+            # @param current_offset [Integer] current message offset
+            # @param low_watermark_offset [Integer]
+            # @param high_watermark_offset [Integer]
+            def initialize(
+              current_offset,
+              low_watermark_offset,
+              high_watermark_offset
+            )
+              @low_watermark_offset = low_watermark_offset
+              @high_watermark_offset = high_watermark_offset
+              @previous_offset = current_offset + 1
+              @current_offset = current_offset
+              @next_offset = current_offset - 1
+              super()
+            end
+
+            # @return [Boolean] show pagination only when there are other things to present
+            def paginate?
+              return true if @current_offset > @low_watermark_offset
+              return true if @current_offset < @high_watermark_offset - 1
+
+              false
+            end
+
+            # @return [Boolean] provide link to the first (newest)
+            def first_offset?
+              @current_offset < @high_watermark_offset - 1
+            end
+
+            # @return [Integer] highest available offset
+            def first_offset
+              @high_watermark_offset - 1
+            end
+
+            # @return [Boolean]
+            def previous_offset?
+              @current_offset < @high_watermark_offset - 1
+            end
+
+            # @return [Boolean] We always show current offset
+            def current_offset?
+              true
+            end
+
+            # @return [String] shows as current page pagination the offset
+            def current_label
+              @current_offset.to_s
+            end
+
+            # @return [Boolean] if not lowest, show
+            def next_offset?
+              @current_offset > @low_watermark_offset
+            end
+
+            # @return [String] params offset key
+            def offset_key
+              'offset'
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/lib/ttl_cache.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      # Non info related extra components used in the UI
+      module Lib
+        # Ttl Cache for caching things in-memory
+        # @note It **is** thread-safe
+        class TtlCache
+          include ::Karafka::Core::Helpers::Time
+
+          # @param ttl [Integer] time in ms how long should this cache keep data
+          def initialize(ttl)
+            @ttl = ttl
+            @times = {}
+            @values = {}
+            @mutex = Mutex.new
+          end
+
+          # Reads data from the cache
+          #
+          # @param key [String, Symbol] key for the cache read
+          # @return [Object] anything that was cached
+          def read(key)
+            @mutex.synchronize do
+              evict
+              @values[key]
+            end
+          end
+
+          # Writes to the cache
+          #
+          # @param key [String, Symbol] key for the cache
+          # @param value [Object] value we want to cache
+          # @return [Object] value we have written
+          def write(key, value)
+            @mutex.synchronize do
+              @times[key] = monotonic_now + @ttl
+              @values[key] = value
+            end
+          end
+
+          # Reads from the cache and if value not present, will run the block and store its result
+          # in the cache
+          #
+          # @param key [String, Symbol] key for the cache read
+          # @return [Object] anything that was cached or yielded
+          def fetch(key)
+            @mutex.synchronize do
+              evict
+
+              return @values[key] if @values.key?(key)
+
+              @values[key] = yield
+            end
+          end
+
+          # Clears the whole cache
+          def clear
+            @mutex.synchronize do
+              @times.clear
+              @values.clear
+            end
+          end
+
+          private
+
+          # Removes expired elements from the cache
+          def evict
+            @times.each do |key, time|
+              next if time >= monotonic_now
+
+              @times.delete(key)
+              @values.delete(key)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/models/cluster_info.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Models
+        # Wraps around the `Karafka::Admin#cluster_info` with caching and some additional aliases
+        # so we can reference relevant information easily
+        class ClusterInfo
+          class << self
+            # Gets us all the cluster metadata info
+            #
+            # @param cached [Boolean] should we use cached data (true by default)
+            # @return [Rdkafka::Metadata] cluster metadata info
+            def fetch(cached: true)
+              cache = ::Karafka::Web.config.ui.cache
+
+              cluster_info = cache.read(:cluster_info)
+
+              if cluster_info.nil? || !cached
+                cluster_info = cache.write(:cluster_info, Karafka::Admin.cluster_info)
+              end
+
+              cluster_info
+            end
+
+            # Returns us all the info about available topics from the cluster
+            #
+            # @param cached [Boolean] should we use cached data (true by default)
+            # @return [Array<Ui::Models::Topic>] topics details
+            def topics(cached: true)
+              fetch(cached: cached)
+                .topics
+                .map { |topic| Topic.new(topic) }
+            end
+
+            # Fetches us details about particular topic
+            #
+            # @param topic_name [String] name of the topic we are looking for
+            # @param cached [Boolean] should we use cached data (true by default)
+            # @return [Ui::Models::Topic] topic details
+            def topic(topic_name, cached: true)
+              topics(cached: cached)
+                .find { |topic_data| topic_data.topic_name == topic_name }
+                .tap { |topic| topic || raise(Web::Errors::Ui::NotFoundError, topic_name) }
+            end
+
+            # @param topic_name [String] name of the topic we are looking for
+            # @param cached [Boolean] should we use cached data (true by default)
+            # @return [Integer] number of partitions in a given topic
+            def partitions_count(topic_name, cached: true)
+              topic(topic_name, cached: cached).partition_count
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/models/consumers_metrics.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Models
+        # Model representing the current consumers metrics most recent state
+        class ConsumersMetrics < Lib::HashProxy
+          class << self
+            # @return [State, false] current consumers metrics or false if not found
+            def current
+              state = fetch
+
+              return false unless state
+
+              # Do not return the state in case web-ui is not enabled because we need our
+              # internal deserializer for it to operate. False will force user to go to the
+              # status page
+              return false unless Models::Status.new.enabled.success?
+
+              state = state.payload
+              new(state)
+            end
+
+            # @return [State] current consumers metrics
+            # @raise [::Karafka::Web::Errors::Ui::NotFoundError] raised when there is no metrics.
+            def current!
+              current || raise(::Karafka::Web::Errors::Ui::NotFoundError)
+            end
+
+            private
+
+            # @return [::Karafka::Messages::Message, nil] most recent state or nil if none
+            def fetch
+              ::Karafka::Admin.read_topic(
+                Karafka::Web.config.topics.consumers.metrics,
+                0,
+                1
+              ).last
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/models/{state.rb → consumers_state.rb}

@@ -7,7 +7,7 @@ module Karafka
         # Represents the current consumer processes aggregated state
         # This state is the core of Karafka reporting. It holds the most important aggregated data
         # as well as pointers to states of particular consumers and their details.
-        class State < Lib::HashProxy
+        class ConsumersState < Lib::HashProxy
           extend ::Karafka::Core::Helpers::Time
 
           class << self
@@ -18,6 +18,10 @@ module Karafka
               state = fetch
 
               return false unless state
+              # Do not return the state in case web-ui is not enabled because we need our
+              # internal deserializer for it to operate. False will force user to go to the
+              # status page
+              return false unless Models::Status.new.enabled.success?
 
               state = state.payload
               evict_expired_processes(state)
@@ -28,7 +32,7 @@ module Karafka
 
             # @return [State] current aggregated state
             # @raise [::Karafka::Web::Errors::Ui::NotFoundError] raised when there is no current
-            #   state. Probably because `karafka server` was never executed
+            #   state.
             def current!
               current || raise(::Karafka::Web::Errors::Ui::NotFoundError)
             end

data/lib/karafka/web/ui/models/health.rb

@@ -12,21 +12,51 @@ module Karafka
             def current(state)
               stats = {}
 
+              fetch_topics_data(state, stats)
+              fetch_rebalance_ages(state, stats)
+
+              stats
+            end
+
+            private
+
+            # Aggregates data on a per topic basis (in the context of a consumer group)
+            # @param state [Hash]
+            # @param stats [Hash] hash where we will store all the aggregated data
+            def fetch_topics_data(state, stats)
               iterate_partitions(state) do |process, consumer_group, topic, partition|
                 cg_name = consumer_group.id
                 t_name = topic.name
                 pt_id = partition.id
 
-                stats[cg_name] ||= {}
-                stats[cg_name][t_name] ||= {}
-                stats[cg_name][t_name][pt_id] = partition.to_h
-                stats[cg_name][t_name][pt_id][:process] = process
+                stats[cg_name] ||= { topics: {} }
+                stats[cg_name][:topics][t_name] ||= {}
+                stats[cg_name][:topics][t_name][pt_id] = partition
+                stats[cg_name][:topics][t_name][pt_id][:process] = process
               end
-
-              stats
             end
 
-            private
+            # Aggregates rebalances ages data
+            # @param state [Hash]
+            # @param stats [Hash] hash where we will store all the aggregated data
+            def fetch_rebalance_ages(state, stats)
+              iterate_partitions(state) do |process, consumer_group|
+                cg_name = consumer_group.id
+                dispatched_at = process.dispatched_at
+
+                ages = consumer_group[:subscription_groups].values.map do |sub_group_details|
+                  rebalance_age_ms = sub_group_details[:state][:rebalance_age] || 0
+                  dispatched_at - rebalance_age_ms / 1_000
+                end
+
+                stats[cg_name][:rebalance_ages] ||= []
+                stats[cg_name][:rebalance_ages] += ages
+              end
+
+              stats.each_value do |details|
+                details[:rebalanced_at] = details[:rebalance_ages].max
+              end
+            end
 
             # Iterates over all partitions, yielding with extra expanded details
             #