karafka-web 0.6.0 → 0.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1f2269efe1b2e14f38c5265d3a8ffd1e7c1bfb1775b11382b27d2cb7a119de03
-  data.tar.gz: e2656bacf8540ea3854eb0ace8a4e5bf83d3aa7c9e5a7a06d1c36bea50083790
+  metadata.gz: d52e64643f448374a2f4efcb5e27089d5f36c8b8cde0eac5e2434c59b07d371c
+  data.tar.gz: '068a2ee3c3d82eeccdde1e001e6788c71980769c58013dd5290ad47002eac47e'
 SHA512:
-  metadata.gz: 665fdedafab36bb818a64c6aca5ea17355baca597878c0d5819c56c266228d1c8c14135c2dd27a8fb21d81c08d6e7958e9b89c8b21e2b8eac03a93519d8c0b77
-  data.tar.gz: 6845812a25677375a2d6931785a0ceb8e1d5ac1a454245d924e271aa8993af733b67697aecc055b65feb799c12a337c1e4969024b903b2130d2f5903c6edef3b
+  metadata.gz: 0ef49085501fafc176d09c6813d3bafc7cb5d56231e2b655462ed12da02effe623a46c1934a8f17e9913dccf8da6405877ddede9a0f519b350613d0fbf4a66a1
+  data.tar.gz: 8251fe9ac27bab1b0992cb8f9baae68fe15100b48814e245589c6a38cc608ee5fa2d65d62faff0fdc8712dcff7685d82a0e87677d56bcc73fd35b07766f02af8

data/CHANGELOG.md

@@ -1,6 +1,27 @@
 # Karafka Web changelog
 
-## 0.6.0 (Unreleased)
+## 0.7.0 (Unreleased)
+- **[Feature]** Introduce per-topic data exploration in the Explorer.
+- [Improvement] Introduce in-memory cluster state cached to improve performance.
+- [Improvement] Switch to offset based pagination instead of per-page pagination.
+- [Improvement] Avoid double-reading of watermark offsets for explorer and errors display.
+- [Improvement] When no params needed for a page, do not include empty params.
+- [Improvement] Do not include page when page is 1 in the url.
+- [Refactor] Reorganize pagination engine to support offset based pagination.
+
+## 0.6.2 (2023-07-22)
+- [Fix] Fix extensive CPU usage when using HPET clock instead of TSC due to interrupt frequency.
+
+## 0.6.1 (2023-06-25)
+- [Improvement] Include the karafka-web version in the status page tags.
+- [Improvement] Report `karafka-web` version that is running in particular processes.
+- [Improvement] Display `karafka-web` version in the per-process view.
+- [Improvement] Report in the web-ui a scenario, where getting cluster info takes more than 500ms as a warning to make people realize, that operating with Kafka with extensive latencies is not recommended.
+- [Improvement] Continue the status assessment flow on warnings.
+- [Fix] Do not recommend running a server as a way to bootstrap the initial state.
+- [Fix] Ensure in the report contract, that `karafka-core`, `karafka-web`, `rdkafka` and `librdkafka` are validated.
+
+## 0.6.0 (2023-06-13)
 - **[Feature]** Introduce producers errors tracking.
 - [Improvement] Display the error origin as a badge to align with consumers view topic assignments.
 - [Improvement] Collect more job metrics for future usage.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka-web (0.6.0)
+    karafka-web (0.6.2)
       erubi (~> 1.4)
       karafka (>= 2.1.4, < 3.0.0)
       karafka-core (>= 2.0.13, < 3.0.0)

data/lib/karafka/web/config.rb

@@ -80,6 +80,8 @@ module Karafka
       end
 
       setting :ui do
+        setting :cache, default: Ui::Lib::TtlCache.new(60_000 * 5)
+
         # Should the payload be decrypted for the Pro Web UI. Default to `false` due to security
         # reasons
         setting :decrypt, default: false

data/lib/karafka/web/tracking/consumers/contracts/report.rb

@@ -12,7 +12,7 @@ module Karafka
           class Report < Tracking::Contracts::Base
             configure
 
-            required(:schema_version) { |val| val.is_a?(String) }
+            required(:schema_version) { |val| val.is_a?(String) && !val.empty? }
             required(:dispatched_at) { |val| val.is_a?(Numeric) && val.positive? }
             # We have consumers and producer reports and need to ensure that each is handled
             # in an expected fashion
@@ -24,7 +24,7 @@ module Karafka
               required(:memory_usage) { |val| val.is_a?(Integer) && val >= 0 }
               required(:memory_total_usage) { |val| val.is_a?(Integer) && val >= 0 }
               required(:memory_size) { |val| val.is_a?(Integer) && val >= 0 }
-              required(:status) { |val| ::Karafka::Status::STATES.key?(val.to_sym) }
+              required(:status) { |val| ::Karafka::Status::STATES.key?(val.to_s.to_sym) }
               required(:listeners) { |val| val.is_a?(Integer) && val >= 0 }
               required(:concurrency) { |val| val.is_a?(Integer) && val.positive? }
               required(:tags) { |val| val.is_a?(Karafka::Core::Taggable::Tags) }
@@ -38,9 +38,13 @@ module Karafka
             end
 
             nested(:versions) do
+              required(:ruby) { |val| val.is_a?(String) && !val.empty? }
               required(:karafka) { |val| val.is_a?(String) && !val.empty? }
+              required(:karafka_core) { |val| val.is_a?(String) && !val.empty? }
+              required(:karafka_web) { |val| val.is_a?(String) && !val.empty? }
               required(:waterdrop) { |val| val.is_a?(String) && !val.empty? }
-              required(:ruby) { |val| val.is_a?(String) && !val.empty? }
+              required(:rdkafka) { |val| val.is_a?(String) && !val.empty? }
+              required(:librdkafka) { |val| val.is_a?(String) && !val.empty? }
             end
 
             nested(:stats) do

data/lib/karafka/web/tracking/consumers/reporter.rb

@@ -93,12 +93,14 @@ module Karafka
           def call
             @running = true
 
+            # We won't track more often anyhow but want to try frequently not to miss a window
+            # We need to convert the sleep interval into seconds for sleep
+            sleep_time = ::Karafka::Web.config.tracking.interval.to_f / 1_000 / 10
+
             loop do
               report
 
-              # We won't track more often anyhow but want to try frequently not to miss a window
-              # We need to convert the sleep interval into seconds for sleep
-              sleep(::Karafka::Web.config.tracking.interval / 1_000 / 10)
+              sleep(sleep_time)
             end
           end
 

data/lib/karafka/web/tracking/consumers/sampler.rb

@@ -75,8 +75,9 @@ module Karafka
               versions: {
                 ruby: ruby_version,
                 karafka: karafka_version,
-                waterdrop: waterdrop_version,
                 karafka_core: karafka_core_version,
+                karafka_web: karafka_web_version,
+                waterdrop: waterdrop_version,
                 rdkafka: rdkafka_version,
                 librdkafka: librdkafka_version
               },

data/lib/karafka/web/tracking/sampler.rb

@@ -32,6 +32,11 @@ module Karafka
           ::Karafka::VERSION
         end
 
+        # @return [String] Karafka Web UI version
+        def karafka_web_version
+          ::Karafka::Web::VERSION
+        end
+
         # @return [String] Karafka::Core version
         def karafka_core_version
           ::Karafka::Core::VERSION

data/lib/karafka/web/ui/base.rb

@@ -68,8 +68,12 @@ module Karafka
         # Allows us to build current path with additional params
         # @param query_data [Hash] query params we want to add to the current path
         path :current do |query_data = {}|
-          q = query_data.map { |k, v| "#{k}=#{CGI.escape(v.to_s)}" }.join('&')
-          "#{request.path}?#{q}"
+          q = query_data
+              .select { |_, v| v }
+              .map { |k, v| "#{k}=#{CGI.escape(v.to_s)}" }
+              .join('&')
+
+          [request.path, q].compact.delete_if(&:empty?).join('?')
         end
 
         # Sets appropriate template variables based on the response object and renders the

data/lib/karafka/web/ui/controllers/base.rb

@@ -12,6 +12,8 @@ module Karafka
             @params = params
           end
 
+          private
+
           # Builds the respond data object with assigned attributes based on instance variables.
           #
           # @return [Responses::Data] data that should be used to render appropriate view
@@ -33,6 +35,21 @@ module Karafka
               attributes
             )
           end
+
+          # Initializes the expected pagination engine and assigns expected arguments
+          # @param args Any arguments accepted by the selected pagination engine
+          def paginate(*args)
+            engine = case args.count
+                     when 2
+                       Ui::Lib::Paginations::PageBased
+                     when 4
+                       Ui::Lib::Paginations::OffsetBased
+                     else
+                       raise ::Karafka::Errors::UnsupportedCaseError, args.count
+                     end
+
+            @pagination = engine.new(*args)
+          end
         end
       end
     end

data/lib/karafka/web/ui/controllers/cluster.rb

@@ -8,7 +8,8 @@ module Karafka
         class Cluster < Base
           # List cluster info data
           def index
-            @cluster_info = Karafka::Admin.cluster_info
+            # Make sure, that for the cluster view we always get the most recent cluster state
+            @cluster_info = Models::ClusterInfo.fetch(cached: false)
 
             partitions_total = []
 
@@ -18,11 +19,13 @@ module Karafka
               end
             end
 
-            @partitions, @next_page = Ui::Lib::PaginateArray.new.call(
+            @partitions, last_page = Ui::Lib::Paginations::Paginators::Arrays.call(
               partitions_total,
               @params.current_page
             )
 
+            paginate(@params.current_page, !last_page)
+
             respond
           end
 

data/lib/karafka/web/ui/controllers/consumers.rb

@@ -11,11 +11,13 @@ module Karafka
           def index
             @current_state = Models::State.current!
             @counters = Models::Counters.new(@current_state)
-            @processes, @next_page = Lib::PaginateArray.new.call(
+            @processes, last_page = Ui::Lib::Paginations::Paginators::Arrays.call(
               Models::Processes.active(@current_state),
               @params.current_page
             )
 
+            paginate(@params.current_page, !last_page)
+
             respond
           end
         end

data/lib/karafka/web/ui/controllers/errors.rb

@@ -10,13 +10,15 @@ module Karafka
         class Errors < Base
           # Lists first page of the errors
           def index
-            @previous_page, @error_messages, @next_page, = Models::Message.page(
-              errors_topic,
-              0,
-              @params.current_page
-            )
-
             @watermark_offsets = Ui::Models::WatermarkOffsets.find(errors_topic, 0)
+            previous_offset, @error_messages, next_offset, = current_page_data
+
+            paginate(
+              previous_offset,
+              @params.current_offset,
+              next_offset,
+              @error_messages.map(&:offset)
+            )
 
             respond
           end
@@ -34,6 +36,17 @@ module Karafka
 
           private
 
+          # @return [Array] Array with requested messages as well as pagination details and other
+          #   obtained metadata
+          def current_page_data
+            Models::Message.offset_page(
+              errors_topic,
+              0,
+              @params.current_offset,
+              @watermark_offsets
+            )
+          end
+
           # @return [String] errors topic
           def errors_topic
             ::Karafka::Web.config.topics.errors

data/lib/karafka/web/ui/controllers/jobs.rb

@@ -19,11 +19,13 @@ module Karafka
               end
             end
 
-            @jobs, @next_page = Ui::Lib::PaginateArray.new.call(
+            @jobs, last_page = Ui::Lib::Paginations::Paginators::Arrays.call(
               jobs_total,
               @params.current_page
             )
 
+            paginate(@params.current_page, !last_page)
+
             respond
           end
         end

data/lib/karafka/web/ui/controllers/requests/params.rb

@@ -22,6 +22,16 @@ module Karafka
                 page.positive? ? page : 1
               end
             end
+
+            # @return [Integer] offset from which we want to start. `-1` indicates, that we want
+            #   to show the first page discovered based on the high watermark offset. If no offset
+            #   is provided, we go with the high offset first page approach
+            def current_offset
+              @current_offset ||= begin
+                offset = @request_params.fetch('offset', -1).to_i
+                offset < -1 ? -1 : offset
+              end
+            end
           end
         end
       end

data/lib/karafka/web/ui/lib/paginations/base.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Lib
+        # Namespace for all the types of pagination engines we want to support
+        module Paginations
+          # Abstraction on top of pagination, so we can alter pagination key and other things
+          # for non-standard pagination views (non page based, etc)
+          #
+          # @note We do not use `_page` explicitly to indicate, that the page scope may not operate
+          #   on numerable pages (1,2,3,4) but can operate on offsets or times, etc. `_offset` is
+          #   more general and may refer to many types of pagination.
+          class Base
+            attr_reader :previous_offset, :current_offset, :next_offset
+
+            # @return [Boolean] Should we show pagination at all
+            def paginate?
+              raise NotImplementedError, 'Implement in a subclass'
+            end
+
+            # @return [Boolean] Should first offset link be active. If false, the first offset link
+            #   will be disabled
+            def first_offset?
+              raise NotImplementedError, 'Implement in a subclass'
+            end
+
+            # @return [String] first offset url value
+            def first_offset
+              raise NotImplementedError, 'Implement in a subclass'
+            end
+
+            # @return [Boolean] Should previous offset link be active. If false, the previous
+            #   offset link will be disabled
+            def previous_offset?
+              raise NotImplementedError, 'Implement in a subclass'
+            end
+
+            # @return [Boolean] Should we show current offset. If false, the current offset link
+            #   will not be visible at all. Useful for non-linear pagination.
+            def current_offset?
+              raise NotImplementedError, 'Implement in a subclass'
+            end
+
+            # @return [Boolean] Should we show next offset pagination. If false, next offset link
+            #   will be disabled.
+            def next_offset?
+              raise NotImplementedError, 'Implement in a subclass'
+            end
+
+            # @return [String] the url offset key
+            def offset_key
+              raise NotImplementedError, 'Implement in a subclass'
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/lib/paginations/offset_based.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Lib
+        module Paginations
+          # Kafka offset based pagination backend
+          #
+          # Allows us to support paginating over offsets
+          class OffsetBased < Base
+            # @param previous_offset [Integer, false] previous offset or false if should not be
+            #   presented
+            # @param current_offset [Integer] current offset
+            # @param next_offset [Integer, Boolean] should we show next offset page button. If
+            #   false it will not be presented.
+            # @param visible_offsets [Array<Integer>] offsets that are visible in the paginated
+            #   view. It is needed for the current page label
+            def initialize(
+              previous_offset,
+              current_offset,
+              next_offset,
+              visible_offsets
+            )
+              @previous_offset = previous_offset
+              @current_offset = current_offset
+              @next_offset = next_offset
+              @visible_offsets = visible_offsets
+              super()
+            end
+
+            # Show pagination only when there is more than one page of results to be presented
+            #
+            # @return [Boolean]
+            def paginate?
+              @current_offset && (!!@previous_offset || !!@next_offset)
+            end
+
+            # @return [Boolean] active only when we are not on the first page. First page is always
+            #   indicated by the current offset being -1. If there is someone that sets up the
+            #   current offset to a value equal to the last message in the topic partition, we do
+            #   not consider it as a first page and we allow to "reset" to -1 via the first page
+            #   button
+            def first_offset?
+              @current_offset != -1
+            end
+
+            # @return [Boolean] first page offset is always nothing because we use the default -1
+            #   for the offset.
+            def first_offset
+              false
+            end
+
+            # @return [Boolean] Active previous page link when it is not the first page
+            def previous_offset?
+              !!@previous_offset
+            end
+
+            # @return [Boolean] We show current label with offsets that are present on the given
+            #   page
+            def current_offset?
+              true
+            end
+
+            # @return [Boolean] move to the next page if not false. False indicates, that there is
+            #   no next page to move to
+            def next_offset?
+              !!@next_offset
+            end
+
+            # If there is no next offset, we point to 0 as there should be no smaller offset than
+            # that in Kafka ever
+            # @return [Integer]
+            def next_offset
+              next_offset? ? @next_offset : 0
+            end
+
+            # @return [String] label of the current page. It is combined out of the first and
+            #   the last offsets to show the range where we are. It will be empty if no offsets
+            #   but this is not a problem as then we should not display pagination at all
+            def current_label
+              first = @visible_offsets.first
+              last = @visible_offsets.last
+              [first, last].compact.uniq.join(' - ').to_s
+            end
+
+            # @return [String] for offset based pagination we use the offset param name
+            def offset_key
+              'offset'
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/lib/paginations/page_based.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Lib
+        module Paginations
+          # Regular page-based pagination engine
+          class PageBased < Base
+            # @param current_offset [Integer] current page
+            # @param show_next_offset [Boolean] should we show next page
+            #   (value is computed automatically)
+            def initialize(
+              current_offset,
+              show_next_offset
+            )
+              @previous_offset = current_offset - 1
+              @current_offset = current_offset
+              @next_offset = show_next_offset ? current_offset + 1 : false
+              super()
+            end
+
+            # Show pagination only when there is more than one page
+            # @return [Boolean]
+            def paginate?
+              @current_offset && (@current_offset > 1 || !!@next_offset)
+            end
+
+            # @return [Boolean] active the first page link when we are not on the first page
+            def first_offset?
+              @current_offset > 1
+            end
+
+            # @return [Boolean] first page for page based pagination is always empty as it moves us
+            #   to the initial page so we do not include any page info
+            def first_offset
+              false
+            end
+
+            # @return [Boolean] Active previous page link when it is not the first page
+            def previous_offset?
+              @current_offset > 1
+            end
+
+            # @return [Boolean] always show current offset pagination value
+            def current_offset?
+              true
+            end
+
+            # @return [String] label of the current page
+            def current_label
+              @current_offset.to_s
+            end
+
+            # @return [Boolean] move to the next page if not false. False indicates, that there is
+            #   no next page to move to
+            def next_offset?
+              @next_offset
+            end
+
+            # @return [String] for page pages pagination, always use page as the url value
+            def offset_key
+              'page'
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/lib/paginations/paginators/arrays.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Lib
+        module Paginations
+          # Namespace for commands that build paginated resources based on the provided page
+          module Paginators
+            # A simple wrapper for paginating array related data structures
+            # We call this with plural (same with `Sets`) to avoid confusion with Ruby classes
+            class Arrays < Base
+              class << self
+                # @param array [Array] array we want to paginate
+                # @param current_page [Integer] page we want to be on
+                # @return [Array<Array, Boolean>] Array with two elements: first is the array with
+                #   data of the given page and second is a boolean flag with info if the elements we got
+                #   are from the last page
+                def call(array, current_page)
+                  slices = array.each_slice(per_page).to_a
+                  current_data = slices[current_page - 1] || []
+                  last_page = !(slices.count >= current_page - 1 && current_data.size >= per_page)
+
+                  [current_data, last_page]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/lib/paginations/paginators/base.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Lib
+        module Paginations
+          module Paginators
+            # Base paginator
+            class Base
+              class << self
+                # @return [Integer] number of elements per page
+                def per_page
+                  ::Karafka::Web.config.ui.per_page
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/lib/paginations/paginators/partitions.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Lib
+        module Paginations
+          module Paginators
+            # Paginator for selecting proper range of partitions for each page
+            # For topics with a lot of partitions we cannot get all the data efficiently, that
+            # is why we limit number of partitions per page and reduce the operations
+            # that way. This allows us to effectively display more while not having to fetch
+            # more partitions then the number of messages per page.
+            # In cases like this we distribute partitions evenly part of partitions on each of
+            # the pages. This may become unreliable for partitions that are not evenly
+            # distributed but this allows us to display data for as many partitions as we want
+            # without overloading the system
+            class Partitions < Base
+              class << self
+                # Computers the partitions slice, materialized page and the limitations status
+                # for a given page
+                # @param partitions_count [Integer] number of partitions for a given topic
+                # @param current_page [Integer] current page
+                # @return [Array<Array<Integer>, Integer, Boolean>] list of partitions that should
+                #   be active on a given page, materialized page for them and info if we had to
+                #   limit the partitions number on a given page
+                def call(partitions_count, current_page)
+                  # How many "chunks" of partitions we will have
+                  slices_count = (partitions_count / per_page.to_f).ceil
+                  # How many partitions in a single slice should we have
+                  in_slice = (partitions_count / slices_count.to_f).ceil
+                  # Which "chunked" page do we want to get
+                  materialized_page = (current_page / slices_count.to_f).ceil
+                  # Which slice is the one we are operating on
+                  active_slice_index = (current_page - 1) % slices_count
+                  # All available slices so we can pick one that is active
+                  partitions_slices = (0...partitions_count).each_slice(in_slice).to_a
+                  # Select active partitions only
+                  active_partitions = partitions_slices[active_slice_index]
+                  # Are we limiting ourselves because of partition count
+                  limited = slices_count > 1
+
+                  [active_partitions, materialized_page, limited]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end