karafka-web 0.6.2 → 0.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d52e64643f448374a2f4efcb5e27089d5f36c8b8cde0eac5e2434c59b07d371c
-  data.tar.gz: '068a2ee3c3d82eeccdde1e001e6788c71980769c58013dd5290ad47002eac47e'
+  metadata.gz: 12e92e2fc2ceb4c30ab825881074948d32c6041d02b882f074a8d2e94ccae7a4
+  data.tar.gz: 3cc07b90ac8037f9509cf7c42f12fa1ce22e1fdca8d67109ed2aef19faa6c4c4
 SHA512:
-  metadata.gz: 0ef49085501fafc176d09c6813d3bafc7cb5d56231e2b655462ed12da02effe623a46c1934a8f17e9913dccf8da6405877ddede9a0f519b350613d0fbf4a66a1
-  data.tar.gz: 8251fe9ac27bab1b0992cb8f9baae68fe15100b48814e245589c6a38cc608ee5fa2d65d62faff0fdc8712dcff7685d82a0e87677d56bcc73fd35b07766f02af8
+  metadata.gz: 11baf2c9e72a216c8cec688922c76dc5d66eee5ac52e0225b0379b490bc5f5b556c9545482d51b7c88eb8ba9a85eec8b74642e98ce8c4c836d44828412638944
+  data.tar.gz: '010109c8fe106e8ecfa27e7ab75e657868ef01e6c9d5677074d70017a193a22475a8ab2c0938ba545d76602d03564f697b99d0802ae02ff75e7e5aed940a0d54'

data/CHANGELOG.md

@@ -1,13 +1,7 @@
 # Karafka Web changelog
 
-## 0.7.0 (Unreleased)
-- **[Feature]** Introduce per-topic data exploration in the Explorer.
-- [Improvement] Introduce in-memory cluster state cached to improve performance.
-- [Improvement] Switch to offset based pagination instead of per-page pagination.
-- [Improvement] Avoid double-reading of watermark offsets for explorer and errors display.
-- [Improvement] When no params needed for a page, do not include empty params.
-- [Improvement] Do not include page when page is 1 in the url.
-- [Refactor] Reorganize pagination engine to support offset based pagination.
+## 0.6.3 (2023-07-22)
+- [Fix] Remove files from 0.7.0 accidentally added to the release.
 
 ## 0.6.2 (2023-07-22)
 - [Fix] Fix extensive CPU usage when using HPET clock instead of TSC due to interrupt frequency.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka-web (0.6.2)
+    karafka-web (0.6.3)
       erubi (~> 1.4)
       karafka (>= 2.1.4, < 3.0.0)
       karafka-core (>= 2.0.13, < 3.0.0)

data/lib/karafka/web/config.rb

@@ -80,8 +80,6 @@ module Karafka
       end
 
       setting :ui do
-        setting :cache, default: Ui::Lib::TtlCache.new(60_000 * 5)
-
         # Should the payload be decrypted for the Pro Web UI. Default to `false` due to security
         # reasons
         setting :decrypt, default: false

data/lib/karafka/web/ui/base.rb

@@ -68,12 +68,8 @@ module Karafka
         # Allows us to build current path with additional params
         # @param query_data [Hash] query params we want to add to the current path
         path :current do |query_data = {}|
-          q = query_data
-              .select { |_, v| v }
-              .map { |k, v| "#{k}=#{CGI.escape(v.to_s)}" }
-              .join('&')
-
-          [request.path, q].compact.delete_if(&:empty?).join('?')
+          q = query_data.map { |k, v| "#{k}=#{CGI.escape(v.to_s)}" }.join('&')
+          "#{request.path}?#{q}"
         end
 
         # Sets appropriate template variables based on the response object and renders the

data/lib/karafka/web/ui/controllers/base.rb

@@ -12,8 +12,6 @@ module Karafka
             @params = params
           end
 
-          private
-
           # Builds the respond data object with assigned attributes based on instance variables.
           #
           # @return [Responses::Data] data that should be used to render appropriate view
@@ -35,21 +33,6 @@ module Karafka
               attributes
             )
           end
-
-          # Initializes the expected pagination engine and assigns expected arguments
-          # @param args Any arguments accepted by the selected pagination engine
-          def paginate(*args)
-            engine = case args.count
-                     when 2
-                       Ui::Lib::Paginations::PageBased
-                     when 4
-                       Ui::Lib::Paginations::OffsetBased
-                     else
-                       raise ::Karafka::Errors::UnsupportedCaseError, args.count
-                     end
-
-            @pagination = engine.new(*args)
-          end
         end
       end
     end

data/lib/karafka/web/ui/controllers/cluster.rb

@@ -8,8 +8,7 @@ module Karafka
         class Cluster < Base
           # List cluster info data
           def index
-            # Make sure, that for the cluster view we always get the most recent cluster state
-            @cluster_info = Models::ClusterInfo.fetch(cached: false)
+            @cluster_info = Karafka::Admin.cluster_info
 
             partitions_total = []
 
@@ -19,13 +18,11 @@ module Karafka
               end
             end
 
-            @partitions, last_page = Ui::Lib::Paginations::Paginators::Arrays.call(
+            @partitions, @next_page = Ui::Lib::PaginateArray.new.call(
               partitions_total,
               @params.current_page
             )
 
-            paginate(@params.current_page, !last_page)
-
             respond
           end
 

data/lib/karafka/web/ui/controllers/consumers.rb

@@ -11,13 +11,11 @@ module Karafka
           def index
             @current_state = Models::State.current!
             @counters = Models::Counters.new(@current_state)
-            @processes, last_page = Ui::Lib::Paginations::Paginators::Arrays.call(
+            @processes, @next_page = Lib::PaginateArray.new.call(
               Models::Processes.active(@current_state),
               @params.current_page
             )
 
-            paginate(@params.current_page, !last_page)
-
             respond
           end
         end

data/lib/karafka/web/ui/controllers/errors.rb

@@ -10,16 +10,14 @@ module Karafka
         class Errors < Base
           # Lists first page of the errors
           def index
-            @watermark_offsets = Ui::Models::WatermarkOffsets.find(errors_topic, 0)
-            previous_offset, @error_messages, next_offset, = current_page_data
-
-            paginate(
-              previous_offset,
-              @params.current_offset,
-              next_offset,
-              @error_messages.map(&:offset)
+            @previous_page, @error_messages, @next_page, = Models::Message.page(
+              errors_topic,
+              0,
+              @params.current_page
             )
 
+            @watermark_offsets = Ui::Models::WatermarkOffsets.find(errors_topic, 0)
+
             respond
           end
 
@@ -36,17 +34,6 @@ module Karafka
 
           private
 
-          # @return [Array] Array with requested messages as well as pagination details and other
-          #   obtained metadata
-          def current_page_data
-            Models::Message.offset_page(
-              errors_topic,
-              0,
-              @params.current_offset,
-              @watermark_offsets
-            )
-          end
-
           # @return [String] errors topic
           def errors_topic
             ::Karafka::Web.config.topics.errors

data/lib/karafka/web/ui/controllers/jobs.rb

@@ -19,13 +19,11 @@ module Karafka
               end
             end
 
-            @jobs, last_page = Ui::Lib::Paginations::Paginators::Arrays.call(
+            @jobs, @next_page = Ui::Lib::PaginateArray.new.call(
               jobs_total,
               @params.current_page
             )
 
-            paginate(@params.current_page, !last_page)
-
             respond
           end
         end

data/lib/karafka/web/ui/controllers/requests/params.rb

@@ -22,16 +22,6 @@ module Karafka
                 page.positive? ? page : 1
               end
             end
-
-            # @return [Integer] offset from which we want to start. `-1` indicates, that we want
-            #   to show the first page discovered based on the high watermark offset. If no offset
-            #   is provided, we go with the high offset first page approach
-            def current_offset
-              @current_offset ||= begin
-                offset = @request_params.fetch('offset', -1).to_i
-                offset < -1 ? -1 : offset
-              end
-            end
           end
         end
       end

data/lib/karafka/web/ui/lib/paginate_array.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Lib
+        # A simple wrapper for paginating array related data structures
+        class PaginateArray
+          # @param array [Array] array we want to paginate
+          # @param current_page [Integer] page we want to be on
+          # @return [Array<Array, <Integer, nil>>] Array with two elements: first is the array with
+          #   data of the given page and second is the next page number of nil in case there is
+          #   no next page (end of data)
+          def call(array, current_page)
+            slices = array.each_slice(per_page).to_a
+
+            current_data = slices[current_page - 1] || []
+
+            if slices.count >= current_page - 1 && current_data.size >= per_page
+              next_page = current_page + 1
+            else
+              next_page = nil
+            end
+
+            [current_data, next_page]
+          end
+
+          private
+
+          # @return [Integer] how many elements should we display in the UI
+          def per_page
+            ::Karafka::Web.config.ui.per_page
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/models/message.rb

@@ -7,8 +7,6 @@ module Karafka
         # A proxy between `::Karafka::Messages::Message` and web UI
         # We work with the Karafka messages but use this model to wrap the work needed.
         class Message
-          extend Lib::Paginations::Paginators
-
           class << self
             # Looks for a message from a given topic partition
             #
@@ -32,44 +30,41 @@ module Karafka
               )
             end
 
-            # Fetches requested `page_count` number of Kafka messages starting from the oldest
-            # requested `start_offset`. If `start_offset` is `-1`, will fetch the most recent
-            # results
+            # Fetches requested page of Kafka messages.
             #
             # @param topic_id [String]
             # @param partition_id [Integer]
-            # @param start_offset [Integer] oldest offset from which we want to get the data
-            # @param watermark_offsets [Ui::Models::WatermarkOffsets] watermark offsets
-            # @return [Array] We return page data as well as all the details needed to build
+            # @param page [Integer]
+            # @return [Array] We return both page data as well as all the details needed to build
             #   the pagination details.
-            def offset_page(topic_id, partition_id, start_offset, watermark_offsets)
-              low_offset = watermark_offsets.low
-              high_offset = watermark_offsets.high
+            def page(topic_id, partition_id, page)
+              low_offset, high_offset = Karafka::Admin.read_watermark_offsets(
+                topic_id,
+                partition_id
+              )
 
-              # If we start from offset -1, it means we want first page with the most recent
-              # results. We obtain this page by using the offset based on the high watermark
-              # off
-              start_offset = high_offset - per_page if start_offset == -1
+              partitions_count = fetch_partition_count(topic_id)
 
-              # No previous pages, no data, and no more offsets
-              no_data_result = [false, [], false]
+              no_data_result = [false, [], false, partitions_count]
 
-              # If there is no data, we return the no results result
+              # If there is not even one message, we need to early exit
+              # If low and high watermark offsets are of the same value, it means no data in the
+              # topic is present
               return no_data_result if low_offset == high_offset
 
+              # We add plus one because we compute previous offset from which we want to start and
+              # not previous page leading offset
+              start_offset = high_offset - (per_page * page)
+
               if start_offset <= low_offset
-                # If this page does not contain max per page, compute how many messages we can
-                # fetch before stopping
                 count = per_page - (low_offset - start_offset)
-                next_offset = false
+                previous_page = page < 2 ? false : page - 1
+                next_page = false
                 start_offset = low_offset
               else
-                next_offset = start_offset - per_page
-                # Do not go below the lowest possible offset
-                next_offset = low_offset if next_offset < low_offset
-                count = high_offset - start_offset
-                # If there would be more messages that we want to get, force max
-                count = per_page if count > per_page
+                previous_page = page < 2 ? false : page - 1
+                next_page = page + 1
+                count = per_page
               end
 
               # This code is a bit tricky. Since topics can be compacted and certain offsets may
@@ -98,97 +93,17 @@ module Karafka
 
                 next unless messages
 
-                previous_offset = start_offset + count
-
                 return [
-                  # If there is a potential previous page with more recent data, compute its
-                  # offset
-                  previous_offset >= high_offset ? false : previous_offset,
-                  fill_compacted(messages, partition_id, context_offset, context_count).reverse,
-                  next_offset
+                  previous_page,
+                  fill_compacted(messages, context_offset, context_count).reverse,
+                  next_page,
+                  partitions_count
                 ]
               end
 
               no_data_result
             end
 
-            # Fetches requested `page_count` number of Kafka messages from the topic partitions
-            # and merges the results. Ensures, that pagination works as expected.
-            #
-            # @param topic_id [String]
-            # @param partitions_ids [Array<Integer>] for which of the partitions we want to
-            #   get the data. This is a limiting factor because of the fact that we have to
-            #   query the watermark offsets independently
-            # @param page [Integer] which page we want to get
-            def topic_page(topic_id, partitions_ids, page)
-              # This is the bottleneck, for each partition we make one request :(
-              offsets = partitions_ids.map do |partition_id|
-                [partition_id, Models::WatermarkOffsets.find(topic_id, partition_id)]
-              end.to_h
-
-              # Count number of elements we have in each partition
-              # This assumes linear presence until low. If not, gaps will be filled like we fill
-              # for per partition view
-              counts = offsets.values.map { |offset| offset[:high] - offset[:low] }
-
-              # Establish initial offsets for the iterator (where to start) per partition
-              # We do not use the negative lookup iterator because we already can compute starting
-              # offsets. This saves a lot of calls to Kafka
-              ranges = Sets.call(counts, page).map do |partition_position, partition_range|
-                partition_id = partitions_ids.to_a[partition_position]
-                watermarks = offsets[partition_id]
-
-                lowest = watermarks[:high] - partition_range.last - 1
-                # We -1 because high watermark offset is the next incoming offset and not the last
-                # one in the topic partition
-                highest = watermarks[:high] - partition_range.first - 1
-
-                # This range represents offsets we want to fetch
-                [partition_id, lowest..highest]
-              end.to_h
-
-              # We start on our topic from the lowest offset for each expected partition
-              iterator = Karafka::Pro::Iterator.new(
-                { topic_id => ranges.transform_values(&:first) }
-              )
-
-              # Build the aggregated representation for each partition messages, so we can start
-              # with assumption that all the topics are fully compacted. Then we can nicely replace
-              # compacted `false` data with real messages, effectively ensuring that the gaps are
-              # filled with `false` out-of-the-box
-              aggregated = Hash.new { |h, k| h[k] = {} }
-
-              # We initialize the hash so we have a constant ascending order based on the partition
-              # number
-              partitions_ids.each { |i| aggregated[i] }
-
-              # We prefill all the potential offsets for each partition, so in case they were
-              # compacted, we get a continuous flow
-              ranges.each do |partition, range|
-                partition_aggr = aggregated[partition]
-                range.each { |i| partition_aggr[i] = [partition, i] }
-              end
-
-              # Iterate over all partitions and collect data
-              iterator.each do |message|
-                range = ranges[message.partition]
-
-                # Do not fetch more data from a partition for which we got last message from the
-                # expected offsets
-                # When all partitions are stopped, we will stop operations. This drastically
-                # improves performance because we no longer have to poll nils
-                iterator.stop_current_partition if message.offset >= range.last
-
-                partition = aggregated[message.partition]
-                partition[message.offset] = message
-              end
-
-              [
-                aggregated.values.map(&:values).map(&:reverse).reduce(:+),
-                !Sets.call(counts, page + 1).empty?
-              ]
-            end
-
             private
 
             # @param args [Object] anything required by the admin `#read_topic`
@@ -202,6 +117,16 @@ module Karafka
               raise
             end
 
+            # @param topic_id [String] id of the topic
+            # @return [Integer] number of partitions this topic has
+            def fetch_partition_count(topic_id)
+              ::Karafka::Admin
+                .cluster_info
+                .topics
+                .find { |topic| topic[:topic_name] == topic_id }
+                .fetch(:partition_count)
+            end
+
             # @return [Integer] elements per page
             def per_page
               ::Karafka::Web.config.ui.per_page
@@ -211,17 +136,16 @@ module Karafka
             # we need to fill those with  just the missing offset and handle this on the UI.
             #
             # @param messages [Array<Karafka::Messages::Message>] selected messages
-            # @param partition_id [Integer] number of partition for which we fill message gap
             # @param start_offset [Integer] offset of the first message (lowest) that we received
             # @param count [Integer] how many messages we wanted - we need that to fill spots to
             #   have exactly the number that was  requested and not more
             # @return [Array<Karafka::Messages::Message, Integer>] array with gaps filled with the
             #   missing offset
-            def fill_compacted(messages, partition_id, start_offset, count)
+            def fill_compacted(messages, start_offset, count)
               Array.new(count) do |index|
                 messages.find do |message|
                   (message.offset - start_offset) == index
-                end || [partition_id, start_offset + index]
+                end || start_offset + index
               end
             end
           end

data/lib/karafka/web/ui/models/status.rb

@@ -197,9 +197,7 @@ module Karafka
           # @note If fails, `connection_time` will be 1_000_000
           def connect
             started = Time.now.to_f
-            # For status we always need uncached data, otherwise status could cache outdated
-            # info
-            @cluster_info = Models::ClusterInfo.fetch(cached: false)
+            @cluster_info = ::Karafka::Admin.cluster_info
             @connection_time = (Time.now.to_f - started) * 1_000
           rescue ::Rdkafka::RdkafkaError
             @connection_time = 1_000_000

data/lib/karafka/web/ui/pro/app.rb

@@ -81,10 +81,6 @@ module Karafka
                 render_response controller.partition(topic_id, partition_id)
               end
 
-              r.get String do |topic_id|
-                render_response controller.topic(topic_id)
-              end
-
               r.get do
                 render_response controller.index
               end
@@ -103,16 +99,12 @@ module Karafka
             r.on 'errors' do
               controller = Controllers::Errors.new(params)
 
-              r.get Integer, Integer do |partition_id, offset|
-                render_response controller.show(partition_id, offset)
-              end
-
               r.get Integer do |partition_id|
-                render_response controller.partition(partition_id)
+                render_response controller.index(partition_id)
               end
 
-              r.get do
-                render_response controller.index
+              r.get Integer, Integer do |partition_id, offset|
+                render_response controller.show(partition_id, offset)
               end
             end
 

data/lib/karafka/web/ui/pro/controllers/consumers.rb

@@ -22,13 +22,11 @@ module Karafka
             def index
               @current_state = Models::State.current!
               @counters = Models::Counters.new(@current_state)
-              @processes, last_page = Lib::Paginations::Paginators::Arrays.call(
+              @processes, @next_page = Lib::PaginateArray.new.call(
                 Models::Processes.active(@current_state),
                 @params.current_page
               )
 
-              paginate(@params.current_page, !last_page)
-
               respond
             end
 

data/lib/karafka/web/ui/pro/controllers/dlq.rb

@@ -28,7 +28,8 @@ module Karafka
                                 .compact
                                 .select(&:itself)
 
-              @dlq_topics = Models::ClusterInfo
+              @dlq_topics = Karafka::Admin
+                            .cluster_info
                             .topics
                             .select { |topic| dlq_topic_names.include?(topic[:topic_name]) }
                             .sort_by { |topic| topic[:topic_name] }

data/lib/karafka/web/ui/pro/controllers/errors.rb

@@ -18,45 +18,18 @@ module Karafka
         module Controllers
           # Errors details controller
           class Errors < Ui::Controllers::Base
-            include Ui::Lib::Paginations
-
-            # Lists all the errors from all the partitions
-            def index
-              @topic_id = errors_topic
-              @partitions_count = Models::ClusterInfo.partitions_count(errors_topic)
-
-              @active_partitions, materialized_page, @limited = Paginators::Partitions.call(
-                @partitions_count, @params.current_page
-              )
-
-              @error_messages, next_page = Models::Message.topic_page(
-                errors_topic, @active_partitions, materialized_page
-              )
-
-              paginate(@params.current_page, next_page)
-
-              respond
-            end
-
             # @param partition_id [Integer] id of the partition of errors we are interested in
-            def partition(partition_id)
+            def index(partition_id)
+              errors_topic = ::Karafka::Web.config.topics.errors
               @partition_id = partition_id
-              @watermark_offsets = Ui::Models::WatermarkOffsets.find(errors_topic, @partition_id)
-              @partitions_count = Models::ClusterInfo.partitions_count(errors_topic)
-
-              previous_offset, @error_messages, next_offset = Models::Message.offset_page(
-                errors_topic,
-                @partition_id,
-                @params.current_offset,
-                @watermark_offsets
-              )
+              @previous_page, @error_messages, @next_page, @partitions_count = \
+                Models::Message.page(
+                  errors_topic,
+                  @partition_id,
+                  @params.current_page
+                )
 
-              paginate(
-                previous_offset,
-                @params.current_offset,
-                next_offset,
-                @error_messages.map(&:offset)
-              )
+              @watermark_offsets = Ui::Models::WatermarkOffsets.find(errors_topic, @partition_id)
 
               respond
             end
@@ -66,6 +39,7 @@ module Karafka
             # @param partition_id [Integer]
             # @param offset [Integer]
             def show(partition_id, offset)
+              errors_topic = ::Karafka::Web.config.topics.errors
               @partition_id = partition_id
               @offset = offset
               @error_message = Models::Message.find(
@@ -76,13 +50,6 @@ module Karafka
 
               respond
             end
-
-            private
-
-            # @return [String] errors topic
-            def errors_topic
-              ::Karafka::Web.config.topics.errors
-            end
           end
         end
       end