karafka-web 0.6.1 → 0.6.2

data/lib/karafka/web/ui/lib/ttl_cache.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      # Non info related extra components used in the UI
+      module Lib
+        # Ttl Cache for caching things in-memory
+        # @note It **is** thread-safe
+        class TtlCache
+          include ::Karafka::Core::Helpers::Time
+
+          # @param ttl [Integer] time in ms how long should this cache keep data
+          def initialize(ttl)
+            @ttl = ttl
+            @times = {}
+            @values = {}
+            @mutex = Mutex.new
+          end
+
+          # Reads data from the cache
+          #
+          # @param key [String, Symbol] key for the cache read
+          # @return [Object] anything that was cached
+          def read(key)
+            @mutex.synchronize do
+              evict
+              @values[key]
+            end
+          end
+
+          # Writes to the cache
+          #
+          # @param key [String, Symbol] key for the cache
+          # @param value [Object] value we want to cache
+          # @return [Object] value we have written
+          def write(key, value)
+            @mutex.synchronize do
+              @times[key] = monotonic_now + @ttl
+              @values[key] = value
+            end
+          end
+
+          # Reads from the cache and if value not present, will run the block and store its result
+          # in the cache
+          #
+          # @param key [String, Symbol] key for the cache read
+          # @return [Object] anything that was cached or yielded
+          def fetch(key)
+            @mutex.synchronize do
+              evict
+
+              return @values[key] if @values.key?(key)
+
+              @values[key] = yield
+            end
+          end
+
+          private
+
+          # Removes expired elements from the cache
+          def evict
+            @times.each do |key, time|
+              next if time >= monotonic_now
+
+              @times.delete(key)
+              @values.delete(key)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/models/cluster_info.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Models
+        # Wraps around the `Karafka::Admin#cluster_info` with caching and some additional aliases
+        # so we can reference relevant information easily
+        class ClusterInfo
+          class << self
+            # Gets us all the cluster metadata info
+            #
+            # @param cached [Boolean] should we use cached data (true by default)
+            # @return [Rdkafka::Metadata] cluster metadata info
+            def fetch(cached: true)
+              cache = ::Karafka::Web.config.ui.cache
+
+              cluster_info = cache.read(:cluster_info)
+
+              if cluster_info.nil? || !cached
+                cluster_info = cache.write(:cluster_info, Karafka::Admin.cluster_info)
+              end
+
+              cluster_info
+            end
+
+            # Returns us all the info about available topics from the cluster
+            #
+            # @param cached [Boolean] should we use cached data (true by default)
+            # @return [Array<Ui::Models::Topic>] topics details
+            def topics(cached: true)
+              fetch(cached: cached)
+                .topics
+                .map { |topic| Topic.new(topic) }
+            end
+
+            # Fetches us details about particular topic
+            #
+            # @param topic_name [String] name of the topic we are looking for
+            # @param cached [Boolean] should we use cached data (true by default)
+            # @return [Ui::Models::Topic] topic details
+            def topic(topic_name, cached: true)
+              topics(cached: cached)
+                .find { |topic_data| topic_data.topic_name == topic_name }
+                .tap { |topic| topic || raise(Web::Errors::Ui::NotFoundError, topic_name) }
+            end
+
+            # @param topic_name [String] name of the topic we are looking for
+            # @param cached [Boolean] should we use cached data (true by default)
+            # @return [Integer] number of partitions in a given topic
+            def partitions_count(topic_name, cached: true)
+              topic(topic_name, cached: cached).partition_count
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/models/message.rb

@@ -7,6 +7,8 @@ module Karafka
         # A proxy between `::Karafka::Messages::Message` and web UI
         # We work with the Karafka messages but use this model to wrap the work needed.
         class Message
+          extend Lib::Paginations::Paginators
+
           class << self
             # Looks for a message from a given topic partition
             #
@@ -30,41 +32,44 @@ module Karafka
               )
             end
 
-            # Fetches requested page of Kafka messages.
+            # Fetches requested `page_count` number of Kafka messages starting from the oldest
+            # requested `start_offset`. If `start_offset` is `-1`, will fetch the most recent
+            # results
             #
             # @param topic_id [String]
             # @param partition_id [Integer]
-            # @param page [Integer]
-            # @return [Array] We return both page data as well as all the details needed to build
+            # @param start_offset [Integer] oldest offset from which we want to get the data
+            # @param watermark_offsets [Ui::Models::WatermarkOffsets] watermark offsets
+            # @return [Array] We return page data as well as all the details needed to build
             #   the pagination details.
-            def page(topic_id, partition_id, page)
-              low_offset, high_offset = Karafka::Admin.read_watermark_offsets(
-                topic_id,
-                partition_id
-              )
+            def offset_page(topic_id, partition_id, start_offset, watermark_offsets)
+              low_offset = watermark_offsets.low
+              high_offset = watermark_offsets.high
 
-              partitions_count = fetch_partition_count(topic_id)
+              # If we start from offset -1, it means we want first page with the most recent
+              # results. We obtain this page by using the offset based on the high watermark
+              # off
+              start_offset = high_offset - per_page if start_offset == -1
 
-              no_data_result = [false, [], false, partitions_count]
+              # No previous pages, no data, and no more offsets
+              no_data_result = [false, [], false]
 
-              # If there is not even one message, we need to early exit
-              # If low and high watermark offsets are of the same value, it means no data in the
-              # topic is present
+              # If there is no data, we return the no results result
               return no_data_result if low_offset == high_offset
 
-              # We add plus one because we compute previous offset from which we want to start and
-              # not previous page leading offset
-              start_offset = high_offset - (per_page * page)
-
               if start_offset <= low_offset
+                # If this page does not contain max per page, compute how many messages we can
+                # fetch before stopping
                 count = per_page - (low_offset - start_offset)
-                previous_page = page < 2 ? false : page - 1
-                next_page = false
+                next_offset = false
                 start_offset = low_offset
               else
-                previous_page = page < 2 ? false : page - 1
-                next_page = page + 1
-                count = per_page
+                next_offset = start_offset - per_page
+                # Do not go below the lowest possible offset
+                next_offset = low_offset if next_offset < low_offset
+                count = high_offset - start_offset
+                # If there would be more messages that we want to get, force max
+                count = per_page if count > per_page
               end
 
               # This code is a bit tricky. Since topics can be compacted and certain offsets may
@@ -93,17 +98,97 @@ module Karafka
 
                 next unless messages
 
+                previous_offset = start_offset + count
+
                 return [
-                  previous_page,
-                  fill_compacted(messages, context_offset, context_count).reverse,
-                  next_page,
-                  partitions_count
+                  # If there is a potential previous page with more recent data, compute its
+                  # offset
+                  previous_offset >= high_offset ? false : previous_offset,
+                  fill_compacted(messages, partition_id, context_offset, context_count).reverse,
+                  next_offset
                 ]
               end
 
               no_data_result
             end
 
+            # Fetches requested `page_count` number of Kafka messages from the topic partitions
+            # and merges the results. Ensures, that pagination works as expected.
+            #
+            # @param topic_id [String]
+            # @param partitions_ids [Array<Integer>] for which of the partitions we want to
+            #   get the data. This is a limiting factor because of the fact that we have to
+            #   query the watermark offsets independently
+            # @param page [Integer] which page we want to get
+            def topic_page(topic_id, partitions_ids, page)
+              # This is the bottleneck, for each partition we make one request :(
+              offsets = partitions_ids.map do |partition_id|
+                [partition_id, Models::WatermarkOffsets.find(topic_id, partition_id)]
+              end.to_h
+
+              # Count number of elements we have in each partition
+              # This assumes linear presence until low. If not, gaps will be filled like we fill
+              # for per partition view
+              counts = offsets.values.map { |offset| offset[:high] - offset[:low] }
+
+              # Establish initial offsets for the iterator (where to start) per partition
+              # We do not use the negative lookup iterator because we already can compute starting
+              # offsets. This saves a lot of calls to Kafka
+              ranges = Sets.call(counts, page).map do |partition_position, partition_range|
+                partition_id = partitions_ids.to_a[partition_position]
+                watermarks = offsets[partition_id]
+
+                lowest = watermarks[:high] - partition_range.last - 1
+                # We -1 because high watermark offset is the next incoming offset and not the last
+                # one in the topic partition
+                highest = watermarks[:high] - partition_range.first - 1
+
+                # This range represents offsets we want to fetch
+                [partition_id, lowest..highest]
+              end.to_h
+
+              # We start on our topic from the lowest offset for each expected partition
+              iterator = Karafka::Pro::Iterator.new(
+                { topic_id => ranges.transform_values(&:first) }
+              )
+
+              # Build the aggregated representation for each partition messages, so we can start
+              # with assumption that all the topics are fully compacted. Then we can nicely replace
+              # compacted `false` data with real messages, effectively ensuring that the gaps are
+              # filled with `false` out-of-the-box
+              aggregated = Hash.new { |h, k| h[k] = {} }
+
+              # We initialize the hash so we have a constant ascending order based on the partition
+              # number
+              partitions_ids.each { |i| aggregated[i] }
+
+              # We prefill all the potential offsets for each partition, so in case they were
+              # compacted, we get a continuous flow
+              ranges.each do |partition, range|
+                partition_aggr = aggregated[partition]
+                range.each { |i| partition_aggr[i] = [partition, i] }
+              end
+
+              # Iterate over all partitions and collect data
+              iterator.each do |message|
+                range = ranges[message.partition]
+
+                # Do not fetch more data from a partition for which we got last message from the
+                # expected offsets
+                # When all partitions are stopped, we will stop operations. This drastically
+                # improves performance because we no longer have to poll nils
+                iterator.stop_current_partition if message.offset >= range.last
+
+                partition = aggregated[message.partition]
+                partition[message.offset] = message
+              end
+
+              [
+                aggregated.values.map(&:values).map(&:reverse).reduce(:+),
+                !Sets.call(counts, page + 1).empty?
+              ]
+            end
+
             private
 
             # @param args [Object] anything required by the admin `#read_topic`
@@ -117,16 +202,6 @@ module Karafka
               raise
             end
 
-            # @param topic_id [String] id of the topic
-            # @return [Integer] number of partitions this topic has
-            def fetch_partition_count(topic_id)
-              ::Karafka::Admin
-                .cluster_info
-                .topics
-                .find { |topic| topic[:topic_name] == topic_id }
-                .fetch(:partition_count)
-            end
-
             # @return [Integer] elements per page
             def per_page
               ::Karafka::Web.config.ui.per_page
@@ -136,16 +211,17 @@ module Karafka
             # we need to fill those with  just the missing offset and handle this on the UI.
             #
             # @param messages [Array<Karafka::Messages::Message>] selected messages
+            # @param partition_id [Integer] number of partition for which we fill message gap
             # @param start_offset [Integer] offset of the first message (lowest) that we received
             # @param count [Integer] how many messages we wanted - we need that to fill spots to
             #   have exactly the number that was  requested and not more
             # @return [Array<Karafka::Messages::Message, Integer>] array with gaps filled with the
             #   missing offset
-            def fill_compacted(messages, start_offset, count)
+            def fill_compacted(messages, partition_id, start_offset, count)
               Array.new(count) do |index|
                 messages.find do |message|
                   (message.offset - start_offset) == index
-                end || start_offset + index
+                end || [partition_id, start_offset + index]
               end
             end
           end

data/lib/karafka/web/ui/models/status.rb

@@ -197,7 +197,9 @@ module Karafka
           # @note If fails, `connection_time` will be 1_000_000
           def connect
             started = Time.now.to_f
-            @cluster_info = ::Karafka::Admin.cluster_info
+            # For status we always need uncached data, otherwise status could cache outdated
+            # info
+            @cluster_info = Models::ClusterInfo.fetch(cached: false)
             @connection_time = (Time.now.to_f - started) * 1_000
           rescue ::Rdkafka::RdkafkaError
             @connection_time = 1_000_000

data/lib/karafka/web/ui/pro/app.rb

@@ -81,6 +81,10 @@ module Karafka
                 render_response controller.partition(topic_id, partition_id)
               end
 
+              r.get String do |topic_id|
+                render_response controller.topic(topic_id)
+              end
+
               r.get do
                 render_response controller.index
               end
@@ -99,12 +103,16 @@ module Karafka
             r.on 'errors' do
               controller = Controllers::Errors.new(params)
 
+              r.get Integer, Integer do |partition_id, offset|
+                render_response controller.show(partition_id, offset)
+              end
+
               r.get Integer do |partition_id|
-                render_response controller.index(partition_id)
+                render_response controller.partition(partition_id)
               end
 
-              r.get Integer, Integer do |partition_id, offset|
-                render_response controller.show(partition_id, offset)
+              r.get do
+                render_response controller.index
               end
             end
 

data/lib/karafka/web/ui/pro/controllers/consumers.rb

@@ -22,11 +22,13 @@ module Karafka
             def index
               @current_state = Models::State.current!
               @counters = Models::Counters.new(@current_state)
-              @processes, @next_page = Lib::PaginateArray.new.call(
+              @processes, last_page = Lib::Paginations::Paginators::Arrays.call(
                 Models::Processes.active(@current_state),
                 @params.current_page
               )
 
+              paginate(@params.current_page, !last_page)
+
               respond
             end
 

data/lib/karafka/web/ui/pro/controllers/dlq.rb

@@ -28,8 +28,7 @@ module Karafka
                                 .compact
                                 .select(&:itself)
 
-              @dlq_topics = Karafka::Admin
-                            .cluster_info
+              @dlq_topics = Models::ClusterInfo
                             .topics
                             .select { |topic| dlq_topic_names.include?(topic[:topic_name]) }
                             .sort_by { |topic| topic[:topic_name] }

data/lib/karafka/web/ui/pro/controllers/errors.rb

@@ -18,18 +18,45 @@ module Karafka
         module Controllers
           # Errors details controller
           class Errors < Ui::Controllers::Base
+            include Ui::Lib::Paginations
+
+            # Lists all the errors from all the partitions
+            def index
+              @topic_id = errors_topic
+              @partitions_count = Models::ClusterInfo.partitions_count(errors_topic)
+
+              @active_partitions, materialized_page, @limited = Paginators::Partitions.call(
+                @partitions_count, @params.current_page
+              )
+
+              @error_messages, next_page = Models::Message.topic_page(
+                errors_topic, @active_partitions, materialized_page
+              )
+
+              paginate(@params.current_page, next_page)
+
+              respond
+            end
+
             # @param partition_id [Integer] id of the partition of errors we are interested in
-            def index(partition_id)
-              errors_topic = ::Karafka::Web.config.topics.errors
+            def partition(partition_id)
               @partition_id = partition_id
-              @previous_page, @error_messages, @next_page, @partitions_count = \
-                Models::Message.page(
-                  errors_topic,
-                  @partition_id,
-                  @params.current_page
-                )
-
               @watermark_offsets = Ui::Models::WatermarkOffsets.find(errors_topic, @partition_id)
+              @partitions_count = Models::ClusterInfo.partitions_count(errors_topic)
+
+              previous_offset, @error_messages, next_offset = Models::Message.offset_page(
+                errors_topic,
+                @partition_id,
+                @params.current_offset,
+                @watermark_offsets
+              )
+
+              paginate(
+                previous_offset,
+                @params.current_offset,
+                next_offset,
+                @error_messages.map(&:offset)
+              )
 
               respond
             end
@@ -39,7 +66,6 @@ module Karafka
             # @param partition_id [Integer]
             # @param offset [Integer]
             def show(partition_id, offset)
-              errors_topic = ::Karafka::Web.config.topics.errors
               @partition_id = partition_id
               @offset = offset
               @error_message = Models::Message.find(
@@ -50,6 +76,13 @@ module Karafka
 
               respond
             end
+
+            private
+
+            # @return [String] errors topic
+            def errors_topic
+              ::Karafka::Web.config.topics.errors
+            end
           end
         end
       end

data/lib/karafka/web/ui/pro/controllers/explorer.rb

@@ -18,10 +18,11 @@ module Karafka
         module Controllers
           # Data explorer controller
           class Explorer < Ui::Controllers::Base
+            include Ui::Lib::Paginations
+
             # Lists all the topics we can explore
             def index
-              @topics = Karafka::Admin
-                        .cluster_info
+              @topics = Models::ClusterInfo
                         .topics
                         .reject { |topic| topic[:topic_name] == '__consumer_offsets' }
                         .sort_by { |topic| topic[:topic_name] }
@@ -29,6 +30,34 @@ module Karafka
               respond
             end
 
+            # Displays aggregated messages from (potentially) all partitions of a topic
+            #
+            # @param topic_id [String]
+            #
+            # @note This view may not be 100% accurate because we merge multiple partitions data
+            #   into a single view and this is never accurate. It can be used however to quickly
+            #   look at most recent data flowing, etc, hence it is still useful for aggregated
+            #   metrics information
+            #
+            # @note We cannot use offset references here because each of the partitions may have
+            #   completely different values
+            def topic(topic_id)
+              @topic_id = topic_id
+              @partitions_count = Models::ClusterInfo.partitions_count(topic_id)
+
+              @active_partitions, materialized_page, @limited = Paginators::Partitions.call(
+                @partitions_count, @params.current_page
+              )
+
+              @messages, next_page = Models::Message.topic_page(
+                topic_id, @active_partitions, materialized_page
+              )
+
+              paginate(@params.current_page, next_page)
+
+              respond
+            end
+
             # Shows messages available in a given partition
             #
             # @param topic_id [String]
@@ -36,13 +65,16 @@ module Karafka
             def partition(topic_id, partition_id)
               @topic_id = topic_id
               @partition_id = partition_id
-
               @watermark_offsets = Ui::Models::WatermarkOffsets.find(topic_id, partition_id)
+              @partitions_count = Models::ClusterInfo.partitions_count(topic_id)
 
-              @previous_page, @messages, @next_page, @partitions_count = Ui::Models::Message.page(
-                @topic_id,
-                @partition_id,
-                @params.current_page
+              previous_offset, @messages, next_offset = current_partition_data
+
+              paginate(
+                previous_offset,
+                @params.current_offset,
+                next_offset,
+                @messages.map(&:offset)
               )
 
               respond
@@ -74,6 +106,19 @@ module Karafka
 
               respond
             end
+
+            private
+
+            # Fetches current page data
+            # @return [Array] fetched data with pagination information for the requested partition
+            def current_partition_data
+              Ui::Models::Message.offset_page(
+                @topic_id,
+                @partition_id,
+                @params.current_offset,
+                @watermark_offsets
+              )
+            end
           end
         end
       end

data/lib/karafka/web/ui/pro/views/errors/_breadcrumbs.erb

@@ -1,14 +1,16 @@
 <li class="breadcrumb-item">
-  <a href="<%= root_path('errors', 0) %>">
+  <a href="<%= root_path('errors') %>">
     Errors
   </a>
 </li>
 
-<li class="breadcrumb-item">
-  <a href="<%= root_path('errors', @partition_id) %>">
-    Partition <%= @partition_id %>
-  </a>
-</li>
+<% if @partition_id %>
+  <li class="breadcrumb-item">
+    <a href="<%= root_path('errors', @partition_id) %>">
+      Partition <%= @partition_id %>
+    </a>
+  </li>
+<% end %>
 
 <% if @offset %>
   <li class="breadcrumb-item">

data/lib/karafka/web/ui/pro/views/errors/_error.erb

@@ -1,4 +1,4 @@
-<% if error_msg.is_a?(Integer) %>
+<% if error_msg.is_a?(Array) %>
   <tr>
     <td colspan="5" class="text-center text-muted">
       This error has either been removed or compacted and is no longer available.

data/lib/karafka/web/ui/pro/views/errors/_partition_option.erb

@@ -3,5 +3,5 @@
 <% else %>
   <option value="<%= root_path('errors', partition) %>">
 <% end %>
-  <%= partition %>
+  <%= partition || 'All' %>
 </option>

data/lib/karafka/web/ui/pro/views/errors/_table.erb

@@ -0,0 +1,21 @@
+<table class="processes bg-white table table-hover table-bordered table-striped mb-0 align-middle">
+  <thead>
+    <tr class="align-middle">
+      <th>Origin</th>
+      <th>Process name</th>
+      <th>Error</th>
+      <th>Occurred at</th>
+      <th></th>
+    </tr>
+  </thead>
+
+  <tbody>
+    <%==
+      each_partial(
+        @error_messages,
+        'errors/error',
+        local: :error_msg
+      )
+    %>
+  </tbody>
+</table>

data/lib/karafka/web/ui/pro/views/errors/_title_with_select.erb

@@ -0,0 +1,31 @@
+<div class="container mb-4">
+  <div class="row">
+    <div class="col">
+      <h3>
+        Errors
+      </h3>
+    </div>
+
+    <div class="col">
+        <div class="col-auto text-end">
+           <label class="col-form-label">Partition</label>
+        </div>
+    </div>
+
+    <div class="col pt-1 mb-0 pb-0">
+        <div class="col-auto">
+          <select class="form-select form-select-sm mb-0 form-control" id="current-partition">
+            <%==
+              each_partial(
+                [nil] + @partitions_count.times.to_a,
+                'errors/partition_option',
+                local: :partition
+              )
+            %>
+          </select>
+        </div>
+    </div>
+  </div>
+
+  <hr>
+</div>