karafka-web 0.6.3 → 0.7.0

data/lib/karafka/web/ui/models/partition.rb

@@ -6,6 +6,33 @@ module Karafka
       module Models
         # Single topic partition data representation model
         class Partition < Lib::HashProxy
+          # @return [Symbol] one of three states in which LSO can be in the correlation to given
+          #   partition in the context of a consumer group.
+          #
+          # @note States descriptions:
+          #   - `:active` all good. No hanging transactions, processing is ok
+          #   - `:at_risk` - there may be hanging transactions but they do not affect processing
+          #     before being stuck. This means, that the transaction still may be finished
+          #     without affecting the processing, hence not having any impact.
+          #   - `:stopped` - we have reached a hanging LSO and we cannot move forward despite more
+          #     data being available. Unless the hanging transaction is killed or it finishes,
+          #     we will not move forward.
+          def lso_risk_state
+            # If last stable is falling behind the high watermark
+            if ls_offset < hi_offset
+              # But it is changing and moving fast enough, it does not mean it is stuck
+              return :active if ls_offset_fd < ::Karafka::Web.config.ui.lso_threshold
+
+              # If it is stuck but we still have work to do, this is not a tragic situation because
+              # maybe it will unstuck before we reach it
+              return :at_risk if (committed_offset || 0) < ls_offset
+
+              # If it is not changing and falling behind high, it is stuck
+              :stopped
+            else
+              :active
+            end
+          end
         end
       end
     end

data/lib/karafka/web/ui/models/process.rb

@@ -40,7 +40,7 @@ module Karafka
               .sort_by(&:started_at)
           end
 
-          # @return [Integer] collective lag on this process
+          # @return [Integer] collective stored lag on this process
           def lag_stored
             consumer_groups
               .flat_map(&:subscription_groups)
@@ -51,6 +51,17 @@ module Karafka
               .sum
           end
 
+          # @return [Integer] collective lag on this process
+          def lag
+            consumer_groups
+              .flat_map(&:subscription_groups)
+              .flat_map(&:topics)
+              .flat_map(&:partitions)
+              .map(&:lag)
+              .delete_if(&:negative?)
+              .sum
+          end
+
           # @return [Integer] number of partitions to which we are currently subscribed
           def subscribed_partitions_count
             consumer_groups

data/lib/karafka/web/ui/models/status.rb

@@ -25,6 +25,7 @@ module Karafka
               when :success then 'successes'
               when :warning then 'warnings'
               when :failure then 'failures'
+              when :halted  then 'failures'
               else
                 raise ::Karafka::Errors::UnsupportedCaseError, status
               end
@@ -36,22 +37,40 @@ module Karafka
             end
           end
 
-          # Initializes the status object and tries to connect to Kafka
-          def initialize
-            connect
+          # Is karafka-web enabled in the `karafka.rb`
+          # Checks if the consumer group for web-ui is injected.
+          # It does **not** check if the group is active because this may depend on the
+          # configuration details, but for the Web-UI web app to work, the routing needs to be
+          # aware of the deserializer, etc
+          def enabled
+            enabled = ::Karafka::App.routes.map(&:name).include?(
+              ::Karafka::Web.config.processing.consumer_group
+            )
+
+            Step.new(
+              enabled ? :success : :failure,
+              nil
+            )
           end
 
           # @return [Status::Step] were we able to connect to Kafka or not and how fast.
           # Some people try to work with Kafka over the internet with really high latency and this
           # should be highlighted in the UI as often the connection just becomes unstable
           def connection
-            level = if @connection_time < 1_000
-                      :success
-                    elsif @connection_time < 1_000_000
-                      :warning
-                    else
-                      :failure
-                    end
+            if enabled.success?
+              # Do not connect more than once during the status object lifetime
+              @connection_time || connect
+
+              level = if @connection_time < 1_000
+                        :success
+                      elsif @connection_time < 1_000_000
+                        :warning
+                      else
+                        :failure
+                      end
+            else
+              level = :halted
+            end
 
             Step.new(
               level,
@@ -81,6 +100,7 @@ module Karafka
               status = :success
               status = :failure if topics_details[topics_consumers_states][:partitions] != 1
               status = :failure if topics_details[topics_consumers_reports][:partitions] != 1
+              status = :failure if topics_details[topics_consumers_metrics][:partitions] != 1
               details = topics_details
             else
               status = :halted
@@ -93,10 +113,31 @@ module Karafka
             )
           end
 
-          # @return [Status::Step] Is the initial state present in the setup or not
-          def initial_state
+          # @return [Status::Step] do we have correct replication for given env
+          def replication
             if partitions.success?
-              @current_state ||= Models::State.current
+              status = :success
+              # low replication is not an error but just a warning and a potential problem
+              # in case of a crash, this is why we do not fail but warn only
+              status = :warning if topics_details.values.any? { |det| det[:replication] < 2 }
+              # Allow for non-production setups to use replication 1 as it is not that relevant
+              status = :success unless Karafka.env.production?
+              details = topics_details
+            else
+              status = :halted
+              details = {}
+            end
+
+            Step.new(
+              status,
+              details
+            )
+          end
+
+          # @return [Status::Step] Is the initial consumers state present in Kafka
+          def initial_consumers_state
+            if replication.success?
+              @current_state ||= Models::ConsumersState.current
               status = @current_state ? :success : :failure
             else
               status = :halted
@@ -108,10 +149,25 @@ module Karafka
             )
           end
 
+          # @return [Status::Step] Is the initial consumers metrics record present in Kafka
+          def initial_consumers_metrics
+            if initial_consumers_state.success?
+              @current_metrics ||= Models::ConsumersMetrics.current
+              status = @current_metrics ? :success : :failure
+            else
+              status = :halted
+            end
+
+            Step.new(
+              status,
+              nil
+            )
+          end
+
           # @return [Status::Step] Is there at least one active karafka server reporting to the
           #   Web UI
           def live_reporting
-            if initial_state.success?
+            if initial_consumers_metrics.success?
               @processes ||= Models::Processes.active(@current_state)
               status = @processes.empty? ? :failure : :success
             else
@@ -128,7 +184,11 @@ module Karafka
           #   consumed actively.
           def state_calculation
             if live_reporting.success?
-              @subscriptions ||= Models::Health.current(@current_state).values.flat_map(&:keys)
+              @subscriptions ||= Models::Health
+                                 .current(@current_state)
+                                 .values.map { |consumer_group| consumer_group[:topics] }
+                                 .flat_map(&:keys)
+
               status = @subscriptions.include?(topics_consumers_reports) ? :success : :failure
             else
               status = :halted
@@ -140,11 +200,26 @@ module Karafka
             )
           end
 
+          # @return [Status::Step] Are we able to actually digest the consumers reports with the
+          #   consumer that is consuming them.
+          def consumers_reports_schema_state
+            status = if state_calculation.success?
+                       @current_state[:schema_state] == 'compatible' ? :success : :failure
+                     else
+                       :halted
+                     end
+
+            Step.new(
+              status,
+              nil
+            )
+          end
+
           # @return [Status::Step] is Pro enabled with all of its features.
           # @note It's not an error not to have it but we want to warn, that some of the features
           #   may not work without Pro.
           def pro_subscription
-            status = if state_calculation.success?
+            status = if consumers_reports_schema_state.success?
                        ::Karafka.pro? ? :success : :warning
                      else
                        :halted
@@ -168,6 +243,11 @@ module Karafka
             ::Karafka::Web.config.topics.consumers.reports.to_s
           end
 
+          # @return [String] consumers metrics topic name
+          def topics_consumers_metrics
+            ::Karafka::Web.config.topics.consumers.metrics.to_s
+          end
+
           # @return [String] errors topic name
           def topics_errors
             ::Karafka::Web.config.topics.errors
@@ -175,10 +255,13 @@ module Karafka
 
           # @return [Hash] hash with topics with which we work details (even if don't exist)
           def topics_details
+            base = { present: false, partitions: 0, replication: 1 }
+
             topics = {
-              topics_consumers_states => { present: false, partitions: 0 },
-              topics_consumers_reports => { present: false, partitions: 0 },
-              topics_errors => { present: false, partitions: 0 }
+              topics_consumers_states => base.dup,
+              topics_consumers_reports => base.dup,
+              topics_consumers_metrics => base.dup,
+              topics_errors => base.dup
             }
 
             @cluster_info.topics.each do |topic|
@@ -186,8 +269,11 @@ module Karafka
 
               next unless topics.key?(name)
 
-              topics[name][:present] = true
-              topics[name][:partitions] = topic[:partition_count]
+              topics[name].merge!(
+                present: true,
+                partitions: topic[:partition_count],
+                replication: topic[:partitions].map { |part| part[:replica_count] }.max
+              )
             end
 
             topics
@@ -197,7 +283,9 @@ module Karafka
           # @note If fails, `connection_time` will be 1_000_000
           def connect
             started = Time.now.to_f
-            @cluster_info = ::Karafka::Admin.cluster_info
+            # For status we always need uncached data, otherwise status could cache outdated
+            # info
+            @cluster_info = Models::ClusterInfo.fetch(cached: false)
             @connection_time = (Time.now.to_f - started) * 1_000
           rescue ::Rdkafka::RdkafkaError
             @connection_time = 1_000_000

data/lib/karafka/web/ui/models/visibility_filter.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Models
+        # Allows for a granular control over what parts of messages are being displayed
+        # There are scenarios where payload or other parts of messages should not be presented
+        # because they may contain sensitive data. This API allows to manage that on a per message
+        # basis.
+        class VisibilityFilter
+          # @param _message [::Karafka::Messages::Message]
+          # @return [Boolean] should message key be visible
+          def key?(_message)
+            true
+          end
+
+          # @param _message [::Karafka::Messages::Message]
+          # @return [Boolean] should message headers be visible
+          def headers?(_message)
+            true
+          end
+
+          # @param message [::Karafka::Messages::Message]
+          # @return [Boolean] should message payload be visible
+          def payload?(message)
+            !message.headers.key?('encryption')
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/pro/app.rb

@@ -32,90 +32,158 @@ module Karafka
           ]
 
           route do |r|
-            r.root { r.redirect root_path('consumers') }
+            r.root { r.redirect root_path('dashboard') }
 
-            @current_page = params.current_page
+            # Serve current version specific assets to prevent users from fetching old assets
+            # after upgrade
+            r.on(:assets, Karafka::Web::VERSION) do
+              r.public
+            end
+
+            r.get 'dashboard' do
+              @breadcrumbs = false
+              controller = Controllers::Dashboard.new(params)
+              controller.index
+            end
 
             r.on 'consumers' do
               controller = Controllers::Consumers.new(params)
 
               r.get String, 'jobs' do |process_id|
-                render_response controller.jobs(process_id)
+                controller.jobs(process_id)
               end
 
               r.get String, 'subscriptions' do |process_id|
-                render_response controller.subscriptions(process_id)
+                controller.subscriptions(process_id)
+              end
+
+              r.get String, 'details' do |process_id|
+                controller.details(process_id)
               end
 
               r.get do
                 @breadcrumbs = false
-                render_response controller.index
+                controller.index
               end
             end
 
             r.get 'jobs' do
               controller = Controllers::Jobs.new(params)
-              render_response controller.index
+              controller.index
             end
 
             r.on 'routing' do
               controller = Controllers::Routing.new(params)
 
               r.get String do |topic_id|
-                render_response controller.show(topic_id)
+                controller.show(topic_id)
               end
 
               r.get do
-                render_response controller.index
+                controller.index
               end
             end
 
             r.on 'explorer' do
               controller = Controllers::Explorer.new(params)
 
+              r.get String, Integer, 'recent' do |topic_id, partition_id|
+                controller.recent(topic_id, partition_id)
+              end
+
+              r.get String, Integer, Integer, 'surrounding' do |topic_id, partition_id, offset|
+                controller.surrounding(topic_id, partition_id, offset)
+              end
+
+              r.get String, 'recent' do |topic_id|
+                controller.recent(topic_id, nil)
+              end
+
+              # Jumps to offset matching the expected time
+              r.get String, Integer, Time do |topic_id, partition_id, time|
+                controller.closest(topic_id, partition_id, time)
+              end
+
               r.get String, Integer, Integer do |topic_id, partition_id, offset|
-                render_response controller.show(topic_id, partition_id, offset)
+                # If when viewing given message we get an offset of different message, we should
+                # redirect there. This allows us to support pagination with the current engine
+                if params.current_offset != -1
+                  r.redirect explorer_path(topic_id, partition_id, params.current_offset)
+                else
+                  controller.show(topic_id, partition_id, offset)
+                end
               end
 
               r.get String, Integer do |topic_id, partition_id|
-                render_response controller.partition(topic_id, partition_id)
+                controller.partition(topic_id, partition_id)
+              end
+
+              r.get String do |topic_id|
+                controller.topic(topic_id)
               end
 
               r.get do
-                render_response controller.index
+                controller.index
               end
             end
 
-            r.get 'health' do
+            r.on 'messages' do
+              controller = Controllers::Messages.new(params)
+
+              r.post String, Integer, Integer, 'republish' do |topic_id, partition_id, offset|
+                controller.republish(topic_id, partition_id, offset)
+              end
+            end
+
+            r.on 'health' do
               controller = Controllers::Health.new(params)
-              render_response controller.index
+
+              r.get 'offsets' do
+                controller.offsets
+              end
+
+              r.get 'overview' do
+                controller.overview
+              end
+
+              r.get do
+                r.redirect root_path('health/overview')
+              end
             end
 
             r.get 'cluster' do
               controller = Controllers::Cluster.new(params)
-              render_response controller.index
+              controller.index
             end
 
             r.on 'errors' do
               controller = Controllers::Errors.new(params)
 
+              r.get Integer, Integer do |partition_id, offset|
+                if params.current_offset != -1
+                  r.redirect root_path('errors', partition_id, params.current_offset)
+                else
+                  controller.show(partition_id, offset)
+                end
+              end
+
               r.get Integer do |partition_id|
-                render_response controller.index(partition_id)
+                controller.partition(partition_id)
               end
 
-              r.get Integer, Integer do |partition_id, offset|
-                render_response controller.show(partition_id, offset)
+              r.get do
+                controller.index
               end
             end
 
             r.get 'dlq' do
               controller = Controllers::Dlq.new(params)
-              render_response controller.index
+              controller.index
             end
 
             r.get 'status' do
               controller = Controllers::Status.new(params)
-              render_response controller.show
+              controller.show
             end
           end
         end

data/lib/karafka/web/ui/pro/controllers/cluster.rb

@@ -1,5 +1,16 @@
 # frozen_string_literal: true
 
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
 module Karafka
   module Web
     module Ui

data/lib/karafka/web/ui/pro/controllers/consumers.rb

@@ -20,29 +20,35 @@ module Karafka
           class Consumers < Ui::Controllers::Base
             # Consumers list
             def index
-              @current_state = Models::State.current!
+              @current_state = Models::ConsumersState.current!
               @counters = Models::Counters.new(@current_state)
-              @processes, @next_page = Lib::PaginateArray.new.call(
+              @processes, last_page = Lib::Paginations::Paginators::Arrays.call(
                 Models::Processes.active(@current_state),
                 @params.current_page
               )
 
+              paginate(@params.current_page, !last_page)
+
               respond
             end
 
             # @param process_id [String] id of the process we're interested in
-            def jobs(process_id)
-              current_state = Models::State.current!
+            def details(process_id)
+              current_state = Models::ConsumersState.current!
               @process = Models::Process.find(current_state, process_id)
 
               respond
             end
 
             # @param process_id [String] id of the process we're interested in
-            def subscriptions(process_id)
-              current_state = Models::State.current!
-              @process = Models::Process.find(current_state, process_id)
+            def jobs(process_id)
+              details(process_id)
+              respond
+            end
 
+            # @param process_id [String] id of the process we're interested in
+            def subscriptions(process_id)
+              details(process_id)
               respond
             end
           end

data/lib/karafka/web/ui/pro/controllers/dashboard.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Ui
+      module Pro
+        module Controllers
+          # Main Karafka Pro Web-Ui dashboard controller
+          class Dashboard < Ui::Controllers::Base
+            # View with statistics dashboard details
+            def index
+              @current_state = Models::ConsumersState.current!
+              @counters = Models::Counters.new(@current_state)
+
+              current_metrics = Models::ConsumersMetrics.current!
+
+              # Build the charts data using the aggregated metrics
+              @aggregated = Models::Metrics::Aggregated.new(
+                current_metrics.to_h.fetch(:aggregated)
+              )
+
+              # Build the charts data about topics using the consumers groups metrics
+              @topics = Models::Metrics::Topics.new(
+                current_metrics.to_h.fetch(:consumer_groups)
+              )
+
+              # Load only historicals for the selected range
+              @aggregated_charts = Models::Metrics::Charts::Aggregated.new(
+                @aggregated, @params.current_range
+              )
+
+              @topics_charts = Models::Metrics::Charts::Topics.new(
+                @topics, @params.current_range
+              )
+
+              respond
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/pro/controllers/dlq.rb

@@ -28,8 +28,7 @@ module Karafka
                                 .compact
                                 .select(&:itself)
 
-              @dlq_topics = Karafka::Admin
-                            .cluster_info
+              @dlq_topics = Models::ClusterInfo
                             .topics
                             .select { |topic| dlq_topic_names.include?(topic[:topic_name]) }
                             .sort_by { |topic| topic[:topic_name] }