karafka-web 0.6.1 → 0.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 25d9203285a7e2587ec78b57bcde9951c647e5eb026c4c32cd7040bfb844ae88
-  data.tar.gz: 46b15b79ebd7b6ce8d9a234c395a6f71bfb7891b8a9cb00d4121fd055bf3f727
+  metadata.gz: d52e64643f448374a2f4efcb5e27089d5f36c8b8cde0eac5e2434c59b07d371c
+  data.tar.gz: '068a2ee3c3d82eeccdde1e001e6788c71980769c58013dd5290ad47002eac47e'
 SHA512:
-  metadata.gz: 529d6eff4eeae68edadd3989cf4e5de9c1c91ea5e0612e2df976717a8700bbbc53e4d917d0421a8e930392b973f19d8a31366be036955392f8ac9a89b821be69
-  data.tar.gz: bcd6de8973599bb034733dd2921cff7277f11aab5e735917695e87722049fcf8bf3a4bcdb6e52a86cbcc4c0dfa91d2ebe7f90c9cc9af3f484ae1b0c6a8dbe8a3
+  metadata.gz: 0ef49085501fafc176d09c6813d3bafc7cb5d56231e2b655462ed12da02effe623a46c1934a8f17e9913dccf8da6405877ddede9a0f519b350613d0fbf4a66a1
+  data.tar.gz: 8251fe9ac27bab1b0992cb8f9baae68fe15100b48814e245589c6a38cc608ee5fa2d65d62faff0fdc8712dcff7685d82a0e87677d56bcc73fd35b07766f02af8

checksums.yaml.gz.sig

@@ -1,5 +1,3 @@
-
-7yʙ|����
-e
-լ�Q:.�^j!�yO5�plZȼ�����Β�>,�5SV����2x�Kw������zq2�m67Ty�M��ٞT��[G��0� )�
-Ͱ�(}���%M��Q�ޯ��؏�Wu�\����P��$3a\{D�!!s��F5�D���,k���
+��lOo�ė����ɝ��"Z�`n�mi��:�Ѿ��9)E���6��S�������I�M<ѓ��(gR:N|Z�C�bd�Z��a�D��wȁƑT�*^�K�$"�
+�UA��i�t�����)�̞L_=^2�n@�M>�W�ĞUm�w����1�|D
+!��b�X��r��=�Vn����
+������?A��+uv����rf$T�U!�t~�	�o[�7гS�?��F��_u

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Karafka Web changelog
 
+## 0.7.0 (Unreleased)
+- **[Feature]** Introduce per-topic data exploration in the Explorer.
+- [Improvement] Introduce in-memory cluster state cached to improve performance.
+- [Improvement] Switch to offset based pagination instead of per-page pagination.
+- [Improvement] Avoid double-reading of watermark offsets for explorer and errors display.
+- [Improvement] When no params needed for a page, do not include empty params.
+- [Improvement] Do not include page when page is 1 in the url.
+- [Refactor] Reorganize pagination engine to support offset based pagination.
+
+## 0.6.2 (2023-07-22)
+- [Fix] Fix extensive CPU usage when using HPET clock instead of TSC due to interrupt frequency.
+
 ## 0.6.1 (2023-06-25)
 - [Improvement] Include the karafka-web version in the status page tags.
 - [Improvement] Report `karafka-web` version that is running in particular processes.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka-web (0.6.1)
+    karafka-web (0.6.2)
       erubi (~> 1.4)
       karafka (>= 2.1.4, < 3.0.0)
       karafka-core (>= 2.0.13, < 3.0.0)

data/lib/karafka/web/config.rb

@@ -80,6 +80,8 @@ module Karafka
       end
 
       setting :ui do
+        setting :cache, default: Ui::Lib::TtlCache.new(60_000 * 5)
+
         # Should the payload be decrypted for the Pro Web UI. Default to `false` due to security
         # reasons
         setting :decrypt, default: false

data/lib/karafka/web/tracking/consumers/reporter.rb

@@ -93,12 +93,14 @@ module Karafka
           def call
             @running = true
 
+            # We won't track more often anyhow but want to try frequently not to miss a window
+            # We need to convert the sleep interval into seconds for sleep
+            sleep_time = ::Karafka::Web.config.tracking.interval.to_f / 1_000 / 10
+
             loop do
               report
 
-              # We won't track more often anyhow but want to try frequently not to miss a window
-              # We need to convert the sleep interval into seconds for sleep
-              sleep(::Karafka::Web.config.tracking.interval / 1_000 / 10)
+              sleep(sleep_time)
             end
           end
 

data/lib/karafka/web/ui/base.rb

@@ -68,8 +68,12 @@ module Karafka
         # Allows us to build current path with additional params
         # @param query_data [Hash] query params we want to add to the current path
         path :current do |query_data = {}|
-          q = query_data.map { |k, v| "#{k}=#{CGI.escape(v.to_s)}" }.join('&')
-          "#{request.path}?#{q}"
+          q = query_data
+              .select { |_, v| v }
+              .map { |k, v| "#{k}=#{CGI.escape(v.to_s)}" }
+              .join('&')
+
+          [request.path, q].compact.delete_if(&:empty?).join('?')
         end
 
         # Sets appropriate template variables based on the response object and renders the

data/lib/karafka/web/ui/controllers/base.rb

@@ -12,6 +12,8 @@ module Karafka
             @params = params
           end
 
+          private
+
           # Builds the respond data object with assigned attributes based on instance variables.
           #
           # @return [Responses::Data] data that should be used to render appropriate view
@@ -33,6 +35,21 @@ module Karafka
               attributes
             )
           end
+
+          # Initializes the expected pagination engine and assigns expected arguments
+          # @param args Any arguments accepted by the selected pagination engine
+          def paginate(*args)
+            engine = case args.count
+                     when 2
+                       Ui::Lib::Paginations::PageBased
+                     when 4
+                       Ui::Lib::Paginations::OffsetBased
+                     else
+                       raise ::Karafka::Errors::UnsupportedCaseError, args.count
+                     end
+
+            @pagination = engine.new(*args)
+          end
         end
       end
     end

data/lib/karafka/web/ui/controllers/cluster.rb

@@ -8,7 +8,8 @@ module Karafka
         class Cluster < Base
           # List cluster info data
           def index
-            @cluster_info = Karafka::Admin.cluster_info
+            # Make sure, that for the cluster view we always get the most recent cluster state
+            @cluster_info = Models::ClusterInfo.fetch(cached: false)
 
             partitions_total = []
 
@@ -18,11 +19,13 @@ module Karafka
               end
             end
 
-            @partitions, @next_page = Ui::Lib::PaginateArray.new.call(
+            @partitions, last_page = Ui::Lib::Paginations::Paginators::Arrays.call(
               partitions_total,
               @params.current_page
             )
 
+            paginate(@params.current_page, !last_page)
+
             respond
           end
 

data/lib/karafka/web/ui/controllers/consumers.rb

@@ -11,11 +11,13 @@ module Karafka
           def index
             @current_state = Models::State.current!
             @counters = Models::Counters.new(@current_state)
-            @processes, @next_page = Lib::PaginateArray.new.call(
+            @processes, last_page = Ui::Lib::Paginations::Paginators::Arrays.call(
               Models::Processes.active(@current_state),
               @params.current_page
             )
 
+            paginate(@params.current_page, !last_page)
+
             respond
           end
         end

data/lib/karafka/web/ui/controllers/errors.rb

@@ -10,13 +10,15 @@ module Karafka
         class Errors < Base
           # Lists first page of the errors
           def index
-            @previous_page, @error_messages, @next_page, = Models::Message.page(
-              errors_topic,
-              0,
-              @params.current_page
-            )
-
             @watermark_offsets = Ui::Models::WatermarkOffsets.find(errors_topic, 0)
+            previous_offset, @error_messages, next_offset, = current_page_data
+
+            paginate(
+              previous_offset,
+              @params.current_offset,
+              next_offset,
+              @error_messages.map(&:offset)
+            )
 
             respond
           end
@@ -34,6 +36,17 @@ module Karafka
 
           private
 
+          # @return [Array] Array with requested messages as well as pagination details and other
+          #   obtained metadata
+          def current_page_data
+            Models::Message.offset_page(
+              errors_topic,
+              0,
+              @params.current_offset,
+              @watermark_offsets
+            )
+          end
+
           # @return [String] errors topic
           def errors_topic
             ::Karafka::Web.config.topics.errors

data/lib/karafka/web/ui/controllers/jobs.rb

@@ -19,11 +19,13 @@ module Karafka
               end
             end
 
-            @jobs, @next_page = Ui::Lib::PaginateArray.new.call(
+            @jobs, last_page = Ui::Lib::Paginations::Paginators::Arrays.call(
               jobs_total,
               @params.current_page
             )
 
+            paginate(@params.current_page, !last_page)
+
             respond
           end
         end

data/lib/karafka/web/ui/controllers/requests/params.rb

@@ -22,6 +22,16 @@ module Karafka
                 page.positive? ? page : 1
               end
             end
+
+            # @return [Integer] offset from which we want to start. `-1` indicates, that we want
+            #   to show the first page discovered based on the high watermark offset. If no offset
+            #   is provided, we go with the high offset first page approach
+            def current_offset
+              @current_offset ||= begin
+                offset = @request_params.fetch('offset', -1).to_i
+                offset < -1 ? -1 : offset
+              end
+            end
           end
         end
       end

data/lib/karafka/web/ui/lib/paginations/base.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Lib
+        # Namespace for all the types of pagination engines we want to support
+        module Paginations
+          # Abstraction on top of pagination, so we can alter pagination key and other things
+          # for non-standard pagination views (non page based, etc)
+          #
+          # @note We do not use `_page` explicitly to indicate, that the page scope may not operate
+          #   on numerable pages (1,2,3,4) but can operate on offsets or times, etc. `_offset` is
+          #   more general and may refer to many types of pagination.
+          class Base
+            attr_reader :previous_offset, :current_offset, :next_offset
+
+            # @return [Boolean] Should we show pagination at all
+            def paginate?
+              raise NotImplementedError, 'Implement in a subclass'
+            end
+
+            # @return [Boolean] Should first offset link be active. If false, the first offset link
+            #   will be disabled
+            def first_offset?
+              raise NotImplementedError, 'Implement in a subclass'
+            end
+
+            # @return [String] first offset url value
+            def first_offset
+              raise NotImplementedError, 'Implement in a subclass'
+            end
+
+            # @return [Boolean] Should previous offset link be active. If false, the previous
+            #   offset link will be disabled
+            def previous_offset?
+              raise NotImplementedError, 'Implement in a subclass'
+            end
+
+            # @return [Boolean] Should we show current offset. If false, the current offset link
+            #   will not be visible at all. Useful for non-linear pagination.
+            def current_offset?
+              raise NotImplementedError, 'Implement in a subclass'
+            end
+
+            # @return [Boolean] Should we show next offset pagination. If false, next offset link
+            #   will be disabled.
+            def next_offset?
+              raise NotImplementedError, 'Implement in a subclass'
+            end
+
+            # @return [String] the url offset key
+            def offset_key
+              raise NotImplementedError, 'Implement in a subclass'
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/lib/paginations/offset_based.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Lib
+        module Paginations
+          # Kafka offset based pagination backend
+          #
+          # Allows us to support paginating over offsets
+          class OffsetBased < Base
+            # @param previous_offset [Integer, false] previous offset or false if should not be
+            #   presented
+            # @param current_offset [Integer] current offset
+            # @param next_offset [Integer, Boolean] should we show next offset page button. If
+            #   false it will not be presented.
+            # @param visible_offsets [Array<Integer>] offsets that are visible in the paginated
+            #   view. It is needed for the current page label
+            def initialize(
+              previous_offset,
+              current_offset,
+              next_offset,
+              visible_offsets
+            )
+              @previous_offset = previous_offset
+              @current_offset = current_offset
+              @next_offset = next_offset
+              @visible_offsets = visible_offsets
+              super()
+            end
+
+            # Show pagination only when there is more than one page of results to be presented
+            #
+            # @return [Boolean]
+            def paginate?
+              @current_offset && (!!@previous_offset || !!@next_offset)
+            end
+
+            # @return [Boolean] active only when we are not on the first page. First page is always
+            #   indicated by the current offset being -1. If there is someone that sets up the
+            #   current offset to a value equal to the last message in the topic partition, we do
+            #   not consider it as a first page and we allow to "reset" to -1 via the first page
+            #   button
+            def first_offset?
+              @current_offset != -1
+            end
+
+            # @return [Boolean] first page offset is always nothing because we use the default -1
+            #   for the offset.
+            def first_offset
+              false
+            end
+
+            # @return [Boolean] Active previous page link when it is not the first page
+            def previous_offset?
+              !!@previous_offset
+            end
+
+            # @return [Boolean] We show current label with offsets that are present on the given
+            #   page
+            def current_offset?
+              true
+            end
+
+            # @return [Boolean] move to the next page if not false. False indicates, that there is
+            #   no next page to move to
+            def next_offset?
+              !!@next_offset
+            end
+
+            # If there is no next offset, we point to 0 as there should be no smaller offset than
+            # that in Kafka ever
+            # @return [Integer]
+            def next_offset
+              next_offset? ? @next_offset : 0
+            end
+
+            # @return [String] label of the current page. It is combined out of the first and
+            #   the last offsets to show the range where we are. It will be empty if no offsets
+            #   but this is not a problem as then we should not display pagination at all
+            def current_label
+              first = @visible_offsets.first
+              last = @visible_offsets.last
+              [first, last].compact.uniq.join(' - ').to_s
+            end
+
+            # @return [String] for offset based pagination we use the offset param name
+            def offset_key
+              'offset'
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/lib/paginations/page_based.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Lib
+        module Paginations
+          # Regular page-based pagination engine
+          class PageBased < Base
+            # @param current_offset [Integer] current page
+            # @param show_next_offset [Boolean] should we show next page
+            #   (value is computed automatically)
+            def initialize(
+              current_offset,
+              show_next_offset
+            )
+              @previous_offset = current_offset - 1
+              @current_offset = current_offset
+              @next_offset = show_next_offset ? current_offset + 1 : false
+              super()
+            end
+
+            # Show pagination only when there is more than one page
+            # @return [Boolean]
+            def paginate?
+              @current_offset && (@current_offset > 1 || !!@next_offset)
+            end
+
+            # @return [Boolean] active the first page link when we are not on the first page
+            def first_offset?
+              @current_offset > 1
+            end
+
+            # @return [Boolean] first page for page based pagination is always empty as it moves us
+            #   to the initial page so we do not include any page info
+            def first_offset
+              false
+            end
+
+            # @return [Boolean] Active previous page link when it is not the first page
+            def previous_offset?
+              @current_offset > 1
+            end
+
+            # @return [Boolean] always show current offset pagination value
+            def current_offset?
+              true
+            end
+
+            # @return [String] label of the current page
+            def current_label
+              @current_offset.to_s
+            end
+
+            # @return [Boolean] move to the next page if not false. False indicates, that there is
+            #   no next page to move to
+            def next_offset?
+              @next_offset
+            end
+
+            # @return [String] for page pages pagination, always use page as the url value
+            def offset_key
+              'page'
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/lib/paginations/paginators/arrays.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Lib
+        module Paginations
+          # Namespace for commands that build paginated resources based on the provided page
+          module Paginators
+            # A simple wrapper for paginating array related data structures
+            # We call this with plural (same with `Sets`) to avoid confusion with Ruby classes
+            class Arrays < Base
+              class << self
+                # @param array [Array] array we want to paginate
+                # @param current_page [Integer] page we want to be on
+                # @return [Array<Array, Boolean>] Array with two elements: first is the array with
+                #   data of the given page and second is a boolean flag with info if the elements we got
+                #   are from the last page
+                def call(array, current_page)
+                  slices = array.each_slice(per_page).to_a
+                  current_data = slices[current_page - 1] || []
+                  last_page = !(slices.count >= current_page - 1 && current_data.size >= per_page)
+
+                  [current_data, last_page]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/lib/paginations/paginators/base.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Lib
+        module Paginations
+          module Paginators
+            # Base paginator
+            class Base
+              class << self
+                # @return [Integer] number of elements per page
+                def per_page
+                  ::Karafka::Web.config.ui.per_page
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/lib/paginations/paginators/partitions.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Lib
+        module Paginations
+          module Paginators
+            # Paginator for selecting proper range of partitions for each page
+            # For topics with a lot of partitions we cannot get all the data efficiently, that
+            # is why we limit number of partitions per page and reduce the operations
+            # that way. This allows us to effectively display more while not having to fetch
+            # more partitions then the number of messages per page.
+            # In cases like this we distribute partitions evenly part of partitions on each of
+            # the pages. This may become unreliable for partitions that are not evenly
+            # distributed but this allows us to display data for as many partitions as we want
+            # without overloading the system
+            class Partitions < Base
+              class << self
+                # Computers the partitions slice, materialized page and the limitations status
+                # for a given page
+                # @param partitions_count [Integer] number of partitions for a given topic
+                # @param current_page [Integer] current page
+                # @return [Array<Array<Integer>, Integer, Boolean>] list of partitions that should
+                #   be active on a given page, materialized page for them and info if we had to
+                #   limit the partitions number on a given page
+                def call(partitions_count, current_page)
+                  # How many "chunks" of partitions we will have
+                  slices_count = (partitions_count / per_page.to_f).ceil
+                  # How many partitions in a single slice should we have
+                  in_slice = (partitions_count / slices_count.to_f).ceil
+                  # Which "chunked" page do we want to get
+                  materialized_page = (current_page / slices_count.to_f).ceil
+                  # Which slice is the one we are operating on
+                  active_slice_index = (current_page - 1) % slices_count
+                  # All available slices so we can pick one that is active
+                  partitions_slices = (0...partitions_count).each_slice(in_slice).to_a
+                  # Select active partitions only
+                  active_partitions = partitions_slices[active_slice_index]
+                  # Are we limiting ourselves because of partition count
+                  limited = slices_count > 1
+
+                  [active_partitions, materialized_page, limited]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/lib/paginations/paginators/sets.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Lib
+        module Paginations
+          module Paginators
+            # Paginator that allows us to take several lists/sets and iterate over them in a
+            # round-robin fashion.
+            #
+            # It does not have to iterate over all the elements from each set for higher pages
+            # making it much more effective than the naive implementation.
+            class Sets < Base
+              class << self
+                # @param counts [Array<Integer>] sets elements counts
+                # @param current_page [Integer] page number
+                # @return [Hash<Integer, Range>] hash with integer keys indicating the count
+                #   location and the range needed to be taken of elements (counting backwards) for
+                #   each partition
+                def call(counts, current_page)
+                  return {} if current_page < 1
+
+                  lists = counts.dup.map.with_index { |el, i| [i, el] }
+
+                  curr_item_index = 0
+                  curr_list_index = 0
+                  items_to_skip_count = per_page * (current_page - 1)
+
+                  loop do
+                    lists_count = lists.length
+                    return {} if lists_count.zero?
+
+                    shortest_list_count = lists.map(&:last).min
+                    mover = (shortest_list_count - curr_item_index)
+                    items_we_are_considering_count = lists_count * mover
+
+                    if items_we_are_considering_count >= items_to_skip_count
+                      curr_item_index += items_to_skip_count / lists_count
+                      curr_list_index = items_to_skip_count % lists_count
+                      break
+                    else
+                      curr_item_index = shortest_list_count
+                      lists.delete_if { |x| x.last == shortest_list_count }
+                      items_to_skip_count -= items_we_are_considering_count
+                    end
+                  end
+
+                  page_items = []
+                  largest_list_count = lists.map(&:last).max
+
+                  while page_items.length < per_page && curr_item_index < largest_list_count
+                    curr_list = lists[curr_list_index]
+
+                    if curr_item_index < curr_list.last
+                      page_items << [curr_list.first, curr_item_index]
+                    end
+
+                    curr_list_index += 1
+                    if curr_list_index == lists.length
+                      curr_list_index = 0
+                      curr_item_index += 1
+                    end
+                  end
+
+                  hashed = Hash.new { |h, k| h[k] = [] }
+
+                  page_items.each do |el|
+                    hashed[el.first] << el.last
+                  end
+
+                  hashed.each do |key, value|
+                    hashed[key] = (value.first..value.last)
+                  end
+
+                  hashed
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end