karafka-web 0.6.0 → 0.6.2

data/lib/karafka/web/ui/lib/paginations/paginators/sets.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Lib
+        module Paginations
+          module Paginators
+            # Paginator that allows us to take several lists/sets and iterate over them in a
+            # round-robin fashion.
+            #
+            # It does not have to iterate over all the elements from each set for higher pages
+            # making it much more effective than the naive implementation.
+            class Sets < Base
+              class << self
+                # @param counts [Array<Integer>] sets elements counts
+                # @param current_page [Integer] page number
+                # @return [Hash<Integer, Range>] hash with integer keys indicating the count
+                #   location and the range needed to be taken of elements (counting backwards) for
+                #   each partition
+                def call(counts, current_page)
+                  return {} if current_page < 1
+
+                  lists = counts.dup.map.with_index { |el, i| [i, el] }
+
+                  curr_item_index = 0
+                  curr_list_index = 0
+                  items_to_skip_count = per_page * (current_page - 1)
+
+                  loop do
+                    lists_count = lists.length
+                    return {} if lists_count.zero?
+
+                    shortest_list_count = lists.map(&:last).min
+                    mover = (shortest_list_count - curr_item_index)
+                    items_we_are_considering_count = lists_count * mover
+
+                    if items_we_are_considering_count >= items_to_skip_count
+                      curr_item_index += items_to_skip_count / lists_count
+                      curr_list_index = items_to_skip_count % lists_count
+                      break
+                    else
+                      curr_item_index = shortest_list_count
+                      lists.delete_if { |x| x.last == shortest_list_count }
+                      items_to_skip_count -= items_we_are_considering_count
+                    end
+                  end
+
+                  page_items = []
+                  largest_list_count = lists.map(&:last).max
+
+                  while page_items.length < per_page && curr_item_index < largest_list_count
+                    curr_list = lists[curr_list_index]
+
+                    if curr_item_index < curr_list.last
+                      page_items << [curr_list.first, curr_item_index]
+                    end
+
+                    curr_list_index += 1
+                    if curr_list_index == lists.length
+                      curr_list_index = 0
+                      curr_item_index += 1
+                    end
+                  end
+
+                  hashed = Hash.new { |h, k| h[k] = [] }
+
+                  page_items.each do |el|
+                    hashed[el.first] << el.last
+                  end
+
+                  hashed.each do |key, value|
+                    hashed[key] = (value.first..value.last)
+                  end
+
+                  hashed
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/lib/ttl_cache.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      # Non info related extra components used in the UI
+      module Lib
+        # Ttl Cache for caching things in-memory
+        # @note It **is** thread-safe
+        class TtlCache
+          include ::Karafka::Core::Helpers::Time
+
+          # @param ttl [Integer] time in ms how long should this cache keep data
+          def initialize(ttl)
+            @ttl = ttl
+            @times = {}
+            @values = {}
+            @mutex = Mutex.new
+          end
+
+          # Reads data from the cache
+          #
+          # @param key [String, Symbol] key for the cache read
+          # @return [Object] anything that was cached
+          def read(key)
+            @mutex.synchronize do
+              evict
+              @values[key]
+            end
+          end
+
+          # Writes to the cache
+          #
+          # @param key [String, Symbol] key for the cache
+          # @param value [Object] value we want to cache
+          # @return [Object] value we have written
+          def write(key, value)
+            @mutex.synchronize do
+              @times[key] = monotonic_now + @ttl
+              @values[key] = value
+            end
+          end
+
+          # Reads from the cache and if value not present, will run the block and store its result
+          # in the cache
+          #
+          # @param key [String, Symbol] key for the cache read
+          # @return [Object] anything that was cached or yielded
+          def fetch(key)
+            @mutex.synchronize do
+              evict
+
+              return @values[key] if @values.key?(key)
+
+              @values[key] = yield
+            end
+          end
+
+          private
+
+          # Removes expired elements from the cache
+          def evict
+            @times.each do |key, time|
+              next if time >= monotonic_now
+
+              @times.delete(key)
+              @values.delete(key)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/models/cluster_info.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Models
+        # Wraps around the `Karafka::Admin#cluster_info` with caching and some additional aliases
+        # so we can reference relevant information easily
+        class ClusterInfo
+          class << self
+            # Gets us all the cluster metadata info
+            #
+            # @param cached [Boolean] should we use cached data (true by default)
+            # @return [Rdkafka::Metadata] cluster metadata info
+            def fetch(cached: true)
+              cache = ::Karafka::Web.config.ui.cache
+
+              cluster_info = cache.read(:cluster_info)
+
+              if cluster_info.nil? || !cached
+                cluster_info = cache.write(:cluster_info, Karafka::Admin.cluster_info)
+              end
+
+              cluster_info
+            end
+
+            # Returns us all the info about available topics from the cluster
+            #
+            # @param cached [Boolean] should we use cached data (true by default)
+            # @return [Array<Ui::Models::Topic>] topics details
+            def topics(cached: true)
+              fetch(cached: cached)
+                .topics
+                .map { |topic| Topic.new(topic) }
+            end
+
+            # Fetches us details about particular topic
+            #
+            # @param topic_name [String] name of the topic we are looking for
+            # @param cached [Boolean] should we use cached data (true by default)
+            # @return [Ui::Models::Topic] topic details
+            def topic(topic_name, cached: true)
+              topics(cached: cached)
+                .find { |topic_data| topic_data.topic_name == topic_name }
+                .tap { |topic| topic || raise(Web::Errors::Ui::NotFoundError, topic_name) }
+            end
+
+            # @param topic_name [String] name of the topic we are looking for
+            # @param cached [Boolean] should we use cached data (true by default)
+            # @return [Integer] number of partitions in a given topic
+            def partitions_count(topic_name, cached: true)
+              topic(topic_name, cached: cached).partition_count
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/models/message.rb

@@ -7,6 +7,8 @@ module Karafka
         # A proxy between `::Karafka::Messages::Message` and web UI
         # We work with the Karafka messages but use this model to wrap the work needed.
         class Message
+          extend Lib::Paginations::Paginators
+
           class << self
             # Looks for a message from a given topic partition
             #
@@ -30,41 +32,44 @@ module Karafka
               )
             end
 
-            # Fetches requested page of Kafka messages.
+            # Fetches requested `page_count` number of Kafka messages starting from the oldest
+            # requested `start_offset`. If `start_offset` is `-1`, will fetch the most recent
+            # results
             #
             # @param topic_id [String]
             # @param partition_id [Integer]
-            # @param page [Integer]
-            # @return [Array] We return both page data as well as all the details needed to build
+            # @param start_offset [Integer] oldest offset from which we want to get the data
+            # @param watermark_offsets [Ui::Models::WatermarkOffsets] watermark offsets
+            # @return [Array] We return page data as well as all the details needed to build
             #   the pagination details.
-            def page(topic_id, partition_id, page)
-              low_offset, high_offset = Karafka::Admin.read_watermark_offsets(
-                topic_id,
-                partition_id
-              )
+            def offset_page(topic_id, partition_id, start_offset, watermark_offsets)
+              low_offset = watermark_offsets.low
+              high_offset = watermark_offsets.high
 
-              partitions_count = fetch_partition_count(topic_id)
+              # If we start from offset -1, it means we want first page with the most recent
+              # results. We obtain this page by using the offset based on the high watermark
+              # off
+              start_offset = high_offset - per_page if start_offset == -1
 
-              no_data_result = [false, [], false, partitions_count]
+              # No previous pages, no data, and no more offsets
+              no_data_result = [false, [], false]
 
-              # If there is not even one message, we need to early exit
-              # If low and high watermark offsets are of the same value, it means no data in the
-              # topic is present
+              # If there is no data, we return the no results result
               return no_data_result if low_offset == high_offset
 
-              # We add plus one because we compute previous offset from which we want to start and
-              # not previous page leading offset
-              start_offset = high_offset - (per_page * page)
-
               if start_offset <= low_offset
+                # If this page does not contain max per page, compute how many messages we can
+                # fetch before stopping
                 count = per_page - (low_offset - start_offset)
-                previous_page = page < 2 ? false : page - 1
-                next_page = false
+                next_offset = false
                 start_offset = low_offset
               else
-                previous_page = page < 2 ? false : page - 1
-                next_page = page + 1
-                count = per_page
+                next_offset = start_offset - per_page
+                # Do not go below the lowest possible offset
+                next_offset = low_offset if next_offset < low_offset
+                count = high_offset - start_offset
+                # If there would be more messages that we want to get, force max
+                count = per_page if count > per_page
               end
 
               # This code is a bit tricky. Since topics can be compacted and certain offsets may
@@ -93,17 +98,97 @@ module Karafka
 
                 next unless messages
 
+                previous_offset = start_offset + count
+
                 return [
-                  previous_page,
-                  fill_compacted(messages, context_offset, context_count).reverse,
-                  next_page,
-                  partitions_count
+                  # If there is a potential previous page with more recent data, compute its
+                  # offset
+                  previous_offset >= high_offset ? false : previous_offset,
+                  fill_compacted(messages, partition_id, context_offset, context_count).reverse,
+                  next_offset
                 ]
               end
 
               no_data_result
             end
 
+            # Fetches requested `page_count` number of Kafka messages from the topic partitions
+            # and merges the results. Ensures, that pagination works as expected.
+            #
+            # @param topic_id [String]
+            # @param partitions_ids [Array<Integer>] for which of the partitions we want to
+            #   get the data. This is a limiting factor because of the fact that we have to
+            #   query the watermark offsets independently
+            # @param page [Integer] which page we want to get
+            def topic_page(topic_id, partitions_ids, page)
+              # This is the bottleneck, for each partition we make one request :(
+              offsets = partitions_ids.map do |partition_id|
+                [partition_id, Models::WatermarkOffsets.find(topic_id, partition_id)]
+              end.to_h
+
+              # Count number of elements we have in each partition
+              # This assumes linear presence until low. If not, gaps will be filled like we fill
+              # for per partition view
+              counts = offsets.values.map { |offset| offset[:high] - offset[:low] }
+
+              # Establish initial offsets for the iterator (where to start) per partition
+              # We do not use the negative lookup iterator because we already can compute starting
+              # offsets. This saves a lot of calls to Kafka
+              ranges = Sets.call(counts, page).map do |partition_position, partition_range|
+                partition_id = partitions_ids.to_a[partition_position]
+                watermarks = offsets[partition_id]
+
+                lowest = watermarks[:high] - partition_range.last - 1
+                # We -1 because high watermark offset is the next incoming offset and not the last
+                # one in the topic partition
+                highest = watermarks[:high] - partition_range.first - 1
+
+                # This range represents offsets we want to fetch
+                [partition_id, lowest..highest]
+              end.to_h
+
+              # We start on our topic from the lowest offset for each expected partition
+              iterator = Karafka::Pro::Iterator.new(
+                { topic_id => ranges.transform_values(&:first) }
+              )
+
+              # Build the aggregated representation for each partition messages, so we can start
+              # with assumption that all the topics are fully compacted. Then we can nicely replace
+              # compacted `false` data with real messages, effectively ensuring that the gaps are
+              # filled with `false` out-of-the-box
+              aggregated = Hash.new { |h, k| h[k] = {} }
+
+              # We initialize the hash so we have a constant ascending order based on the partition
+              # number
+              partitions_ids.each { |i| aggregated[i] }
+
+              # We prefill all the potential offsets for each partition, so in case they were
+              # compacted, we get a continuous flow
+              ranges.each do |partition, range|
+                partition_aggr = aggregated[partition]
+                range.each { |i| partition_aggr[i] = [partition, i] }
+              end
+
+              # Iterate over all partitions and collect data
+              iterator.each do |message|
+                range = ranges[message.partition]
+
+                # Do not fetch more data from a partition for which we got last message from the
+                # expected offsets
+                # When all partitions are stopped, we will stop operations. This drastically
+                # improves performance because we no longer have to poll nils
+                iterator.stop_current_partition if message.offset >= range.last
+
+                partition = aggregated[message.partition]
+                partition[message.offset] = message
+              end
+
+              [
+                aggregated.values.map(&:values).map(&:reverse).reduce(:+),
+                !Sets.call(counts, page + 1).empty?
+              ]
+            end
+
             private
 
             # @param args [Object] anything required by the admin `#read_topic`
@@ -117,16 +202,6 @@ module Karafka
               raise
             end
 
-            # @param topic_id [String] id of the topic
-            # @return [Integer] number of partitions this topic has
-            def fetch_partition_count(topic_id)
-              ::Karafka::Admin
-                .cluster_info
-                .topics
-                .find { |topic| topic[:topic_name] == topic_id }
-                .fetch(:partition_count)
-            end
-
             # @return [Integer] elements per page
             def per_page
               ::Karafka::Web.config.ui.per_page
@@ -136,16 +211,17 @@ module Karafka
             # we need to fill those with  just the missing offset and handle this on the UI.
             #
             # @param messages [Array<Karafka::Messages::Message>] selected messages
+            # @param partition_id [Integer] number of partition for which we fill message gap
             # @param start_offset [Integer] offset of the first message (lowest) that we received
             # @param count [Integer] how many messages we wanted - we need that to fill spots to
             #   have exactly the number that was  requested and not more
             # @return [Array<Karafka::Messages::Message, Integer>] array with gaps filled with the
             #   missing offset
-            def fill_compacted(messages, start_offset, count)
+            def fill_compacted(messages, partition_id, start_offset, count)
               Array.new(count) do |index|
                 messages.find do |message|
                   (message.offset - start_offset) == index
-                end || start_offset + index
+                end || [partition_id, start_offset + index]
               end
             end
           end

data/lib/karafka/web/ui/models/status.rb

@@ -5,6 +5,7 @@ module Karafka
     module Ui
       module Models
         # Model that represents the general status of the Web UI.
+        #
         # We use this data to display a status page that helps with debugging on what is missing
         # in the overall setup of the Web UI.
         #
@@ -15,7 +16,18 @@ module Karafka
           Step = Struct.new(:status, :details) do
             # @return [Boolean] is the given step successfully configured and working
             def success?
-              status == :success
+              status == :success || status == :warning
+            end
+
+            # @return [String] local namespace for partial of a given type
+            def partial_namespace
+              case status
+              when :success then 'successes'
+              when :warning then 'warnings'
+              when :failure then 'failures'
+              else
+                raise ::Karafka::Errors::UnsupportedCaseError, status
+              end
             end
 
             # @return [String] stringified status
@@ -29,11 +41,21 @@ module Karafka
             connect
           end
 
-          # @return [Status::Step] were we able to connect to Kafka or not
+          # @return [Status::Step] were we able to connect to Kafka or not and how fast.
+          # Some people try to work with Kafka over the internet with really high latency and this
+          # should be highlighted in the UI as often the connection just becomes unstable
           def connection
+            level = if @connection_time < 1_000
+                      :success
+                    elsif @connection_time < 1_000_000
+                      :warning
+                    else
+                      :failure
+                    end
+
             Step.new(
-              @connected ? :success : :failure,
-              nil
+              level,
+              { time: @connection_time }
             )
           end
 
@@ -171,12 +193,16 @@ module Karafka
             topics
           end
 
-          # Tries connecting with the cluster and sets the connection state
+          # Tries connecting with the cluster and saves the cluster info and the connection time
+          # @note If fails, `connection_time` will be 1_000_000
           def connect
-            @cluster_info = ::Karafka::Admin.cluster_info
-            @connected = true
+            started = Time.now.to_f
+            # For status we always need uncached data, otherwise status could cache outdated
+            # info
+            @cluster_info = Models::ClusterInfo.fetch(cached: false)
+            @connection_time = (Time.now.to_f - started) * 1_000
           rescue ::Rdkafka::RdkafkaError
-            @connected = false
+            @connection_time = 1_000_000
           end
         end
       end

data/lib/karafka/web/ui/pro/app.rb

@@ -81,6 +81,10 @@ module Karafka
                 render_response controller.partition(topic_id, partition_id)
               end
 
+              r.get String do |topic_id|
+                render_response controller.topic(topic_id)
+              end
+
               r.get do
                 render_response controller.index
               end
@@ -99,12 +103,16 @@ module Karafka
             r.on 'errors' do
               controller = Controllers::Errors.new(params)
 
+              r.get Integer, Integer do |partition_id, offset|
+                render_response controller.show(partition_id, offset)
+              end
+
               r.get Integer do |partition_id|
-                render_response controller.index(partition_id)
+                render_response controller.partition(partition_id)
               end
 
-              r.get Integer, Integer do |partition_id, offset|
-                render_response controller.show(partition_id, offset)
+              r.get do
+                render_response controller.index
               end
             end
 

data/lib/karafka/web/ui/pro/controllers/consumers.rb

@@ -22,11 +22,13 @@ module Karafka
             def index
               @current_state = Models::State.current!
               @counters = Models::Counters.new(@current_state)
-              @processes, @next_page = Lib::PaginateArray.new.call(
+              @processes, last_page = Lib::Paginations::Paginators::Arrays.call(
                 Models::Processes.active(@current_state),
                 @params.current_page
               )
 
+              paginate(@params.current_page, !last_page)
+
               respond
             end
 

data/lib/karafka/web/ui/pro/controllers/dlq.rb

@@ -28,8 +28,7 @@ module Karafka
                                 .compact
                                 .select(&:itself)
 
-              @dlq_topics = Karafka::Admin
-                            .cluster_info
+              @dlq_topics = Models::ClusterInfo
                             .topics
                             .select { |topic| dlq_topic_names.include?(topic[:topic_name]) }
                             .sort_by { |topic| topic[:topic_name] }

data/lib/karafka/web/ui/pro/controllers/errors.rb

@@ -18,18 +18,45 @@ module Karafka
         module Controllers
           # Errors details controller
           class Errors < Ui::Controllers::Base
+            include Ui::Lib::Paginations
+
+            # Lists all the errors from all the partitions
+            def index
+              @topic_id = errors_topic
+              @partitions_count = Models::ClusterInfo.partitions_count(errors_topic)
+
+              @active_partitions, materialized_page, @limited = Paginators::Partitions.call(
+                @partitions_count, @params.current_page
+              )
+
+              @error_messages, next_page = Models::Message.topic_page(
+                errors_topic, @active_partitions, materialized_page
+              )
+
+              paginate(@params.current_page, next_page)
+
+              respond
+            end
+
             # @param partition_id [Integer] id of the partition of errors we are interested in
-            def index(partition_id)
-              errors_topic = ::Karafka::Web.config.topics.errors
+            def partition(partition_id)
               @partition_id = partition_id
-              @previous_page, @error_messages, @next_page, @partitions_count = \
-                Models::Message.page(
-                  errors_topic,
-                  @partition_id,
-                  @params.current_page
-                )
-
               @watermark_offsets = Ui::Models::WatermarkOffsets.find(errors_topic, @partition_id)
+              @partitions_count = Models::ClusterInfo.partitions_count(errors_topic)
+
+              previous_offset, @error_messages, next_offset = Models::Message.offset_page(
+                errors_topic,
+                @partition_id,
+                @params.current_offset,
+                @watermark_offsets
+              )
+
+              paginate(
+                previous_offset,
+                @params.current_offset,
+                next_offset,
+                @error_messages.map(&:offset)
+              )
 
               respond
             end
@@ -39,7 +66,6 @@ module Karafka
             # @param partition_id [Integer]
             # @param offset [Integer]
             def show(partition_id, offset)
-              errors_topic = ::Karafka::Web.config.topics.errors
               @partition_id = partition_id
               @offset = offset
               @error_message = Models::Message.find(
@@ -50,6 +76,13 @@ module Karafka
 
               respond
             end
+
+            private
+
+            # @return [String] errors topic
+            def errors_topic
+              ::Karafka::Web.config.topics.errors
+            end
           end
         end
       end