karafka-web 0.10.3 → 0.11.0.beta1

data/lib/karafka/web/tracking/consumers/reporter.rb

@@ -63,7 +63,7 @@ module Karafka
               # Report consumers statuses
               messages = [
                 {
-                  topic: ::Karafka::Web.config.topics.consumers.reports,
+                  topic: ::Karafka::Web.config.topics.consumers.reports.name,
                   payload: Zlib::Deflate.deflate(report.to_json),
                   key: process_id,
                   partition: 0,
@@ -76,7 +76,7 @@ module Karafka
                 @error_contract.validate!(error)
 
                 {
-                  topic: Karafka::Web.config.topics.errors,
+                  topic: Karafka::Web.config.topics.errors.name,
                   payload: Zlib::Deflate.deflate(error.to_json),
                   # Always dispatch errors from the same process to the same partition
                   key: process_id,

data/lib/karafka/web/tracking/consumers/sampler.rb

@@ -15,7 +15,7 @@ module Karafka
           # Current schema version
           # This is used for detecting incompatible changes and not using outdated data during
           # upgrades
-          SCHEMA_VERSION = '1.4.0'
+          SCHEMA_VERSION = '1.4.1'
 
           # Counters that count events occurrences during the given window
           COUNTERS_BASE = {
@@ -51,7 +51,17 @@ module Karafka
             @subscription_groups = Hash.new do |h, sg_id|
               h[sg_id] = {
                 id: sg_id,
-                polled_at: monotonic_now
+                polled_at: monotonic_now,
+                topics: Hash.new do |h1, topic|
+                  h1[topic] = Hash.new do |h2, partition|
+                    # We track those details in case we need to fill statistical gaps for
+                    # transactional consumers
+                    h2[partition] = {
+                      seek_offset: -1,
+                      transactional: false
+                    }
+                  end
+                end
               }
             end
 
@@ -223,11 +233,9 @@ module Karafka
           def memory_size
             @memory_size ||= case RUBY_PLATFORM
                              when /linux/
-                               @shell
-                             .call('grep MemTotal /proc/meminfo')
-                             .match(/(\d+)/)
-                             .to_s
-                             .to_i
+                               mem_info = File.read('/proc/meminfo')
+                               mem_total_line = mem_info.match(/MemTotal:\s*(?<total>\d+)/)
+                               mem_total_line['total'].to_i
                              when /darwin|bsd/
                                @shell
                              .call('sysctl -a')
@@ -245,7 +253,13 @@ module Karafka
           # @return [Array<Float>] load averages for last 1, 5 and 15 minutes
           def cpu_usage
             case RUBY_PLATFORM
-            when /darwin|bsd|linux/
+            when /linux/
+              File
+                .read('/proc/loadavg')
+                .split(' ')
+                .first(3)
+                .map(&:to_f)
+            when /darwin|bsd/
               @shell
                 .call('w | head -1')
                 .strip
@@ -280,10 +294,21 @@ module Karafka
           def memory_threads_ps
             @memory_threads_ps = case RUBY_PLATFORM
                                  when /linux/
-                                   @shell
-                                 .call('ps -A -o rss=,thcount=,pid=')
-                                 .split("\n")
-                                 .map { |row| row.strip.split(' ').map(&:to_i) }
+                                   page_size = Karafka::Web::Tracking::Helpers::Sysconf.page_size
+                                   status_file = "/proc/#{::Process.pid}/status"
+
+                                   pid = status_file.match(%r{/proc/(\d+)/status})[1]
+
+                                   # Extract thread count from /proc/<pid>/status
+                                   thcount = File.read(status_file)[/^Threads:\s+(\d+)/, 1].to_i
+
+                                   # Extract RSS from /proc/<pid>/statm (second field)
+                                   statm_file = "/proc/#{pid}/statm"
+                                   rss_pages = File.read(statm_file).split[1].to_i rescue 0
+                                   # page size is retrieved from Sysconf
+                                   rss_kb = (rss_pages * page_size) / 1024
+
+                                   [[rss_kb, thcount, pid.to_i]]
                                  # thcount is not available on macos ps
                                  # because of that we inject 0 as threads count similar to how
                                  # we do on windows
@@ -308,9 +333,51 @@ module Karafka
                   # This should be always available, since the subscription group polled at time
                   # is first initialized before we start polling, there should be no case where
                   # we have statistics about a given subscription group but we do not have the
-                  # last polling time
-                  polled_at = subscription_groups.fetch(sg_id).fetch(:polled_at)
+                  # sg reference
+                  sg_tracking = subscription_groups.fetch(sg_id)
+
+                  polled_at = sg_tracking.fetch(:polled_at)
                   sg_details[:state][:poll_age] = (monotonic_now - polled_at).round(2)
+
+                  sg_details[:topics].each do |topic_name, topic_details|
+                    topic_details[:partitions].each do |partition_id, partition_details|
+                      # Always assume non-transactional as default. Will be overwritten by the
+                      # consumer level details if collected
+                      partition_details[:transactional] ||= false
+
+                      # If we have stored offset or stored lag, it means it's not a transactional
+                      # consumer at all so we can skip enrichment
+                      next if partition_details[:lag_stored].positive?
+                      next if partition_details[:stored_offset].positive?
+                      next unless sg_tracking[:topics].key?(topic_name)
+                      next unless sg_tracking[:topics][topic_name].key?(partition_id)
+
+                      k_partition_details = sg_tracking[:topics][topic_name][partition_id]
+
+                      # If seek offset was not yey set, nothing to enrich
+                      next unless k_partition_details[:seek_offset].positive?
+
+                      partition_details[:transactional] = k_partition_details[:transactional]
+
+                      # Seek offset is always +1 from the last stored in Karafka
+                      seek_offset = k_partition_details[:seek_offset]
+                      stored_offset = seek_offset - 1
+
+                      # In case of transactions we have to compute the lag ourselves
+                      # -1 because ls offset (or high watermark) is last + 1
+                      lag = partition_details[:ls_offset] - seek_offset
+                      # This can happen if ls_offset is refreshed slower than our stored offset
+                      # fetching from Karafka transactional layer
+                      lag = 0 if lag.negative?
+
+                      partition_details[:lag] = lag
+                      partition_details[:lag_d] = 0
+                      partition_details[:lag_stored] = lag
+                      partition_details[:lag_stored_d] = 0
+                      partition_details[:stored_offset] = stored_offset
+                      partition_details[:committed_offset] = stored_offset
+                    end
+                  end
                 end
               end
             end

data/lib/karafka/web/tracking/helpers/sysconf.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      # Namespace for tracking related helpers
+      module Helpers
+        # Namespace for unix-based helper methods used to fetch OS details
+        module Sysconf
+          extend FFI::Library
+
+          case RUBY_PLATFORM
+          when /linux/
+            ffi_lib 'libc.so.6' # Standard C library on Linux
+            SC_PAGESIZE = 30 # _SC_PAGESIZE constant
+          when /darwin/
+            ffi_lib 'libSystem.B.dylib' # Standard C library on macOS
+            SC_PAGESIZE = 29 # _SC_PAGESIZE constant
+          end
+
+          attach_function :sysconf, [:int], :long
+
+          class << self
+            # @return [Integer]
+            def page_size
+              sysconf(SC_PAGESIZE)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/producers/reporter.rb

@@ -10,13 +10,6 @@ module Karafka
         #   because there is no expectation on immediate status updates for producers and their
         #   dispatch flow is always periodic based.
         class Reporter < Tracking::Reporter
-          # Minimum number of messages to produce to produce them in sync mode
-          # This acts as a small back-off not to overload the system in case we would have
-          # extremely big number of errors happening
-          PRODUCE_SYNC_THRESHOLD = 25
-
-          private_constant :PRODUCE_SYNC_THRESHOLD
-
           # This mutex is shared between tracker and samplers so there is no case where metrics
           # would be collected same time tracker reports
           MUTEX = Mutex.new
@@ -40,7 +33,7 @@ module Karafka
                 @error_contract.validate!(error)
 
                 {
-                  topic: Karafka::Web.config.topics.errors,
+                  topic: Karafka::Web.config.topics.errors.name,
                   payload: error.to_json,
                   # Always dispatch errors from the same process to the same partition
                   key: error[:process][:id]
@@ -82,7 +75,7 @@ module Karafka
           #   normal operations we should not have that many messages to dispatch and it should not
           #   slowdown any processing.
           def produce(messages)
-            if messages.count >= PRODUCE_SYNC_THRESHOLD
+            if messages.count >= ::Karafka::Web.config.tracking.producers.sync_threshold
               ::Karafka::Web.producer.produce_many_sync(messages)
             else
               ::Karafka::Web.producer.produce_many_async(messages)

data/lib/karafka/web/ui/app.rb

@@ -11,122 +11,29 @@ module Karafka
 
         instance_exec(&CONTEXT_DETAILS)
 
+        # Sub-routes for given pieces of the Web UI
+        SUB_ROUTES = [
+          Routes::Assets,
+          Routes::Dashboard,
+          Routes::Consumers,
+          Routes::ProOnly,
+          Routes::Jobs,
+          Routes::Routing,
+          Routes::Cluster,
+          Routes::Errors,
+          Routes::Status,
+          Routes::Support,
+          Routes::Ux
+        ].freeze
+
+        private_constant :SUB_ROUTES
+
         route do |r|
           r.root { r.redirect root_path('dashboard') }
 
-          # Serve current version specific assets to prevent users from fetching old assets
-          # after upgrade
-          r.on 'assets', Karafka::Web::VERSION do
-            r.public
-          end
-
-          r.get 'dashboard' do
-            @breadcrumbs = false
-            controller = Controllers::DashboardController.new(params)
-            controller.index
-          end
-
-          r.on 'consumers' do
-            %w[
-              performance
-              controls
-              commands
-            ].each do |path|
-              r.get path do |_process_id|
-                raise Errors::Ui::ProOnlyError
-              end
-            end
-
-            r.get String, 'subscriptions' do |_process_id|
-              raise Errors::Ui::ProOnlyError
-            end
-
-            r.get do
-              controller = Controllers::ConsumersController.new(params)
-              controller.index
-            end
-          end
-
-          %w[
-            health
-            explorer
-            dlq
-          ].each do |route|
-            r.get route, [String, true], [String, true] do
-              raise Errors::Ui::ProOnlyError
-            end
-          end
-
-          r.on 'jobs' do
-            controller = Controllers::JobsController.new(params)
-
-            r.get 'running' do
-              controller.running
-            end
-
-            r.get 'pending' do
-              controller.pending
-            end
-
-            r.redirect root_path('jobs/running')
-          end
-
-          r.on 'routing' do
-            controller = Controllers::RoutingController.new(params)
-
-            r.get String do |topic_id|
-              controller.show(topic_id)
-            end
-
-            r.get do
-              controller.index
-            end
-          end
-
-          r.on 'cluster' do
-            controller = Controllers::ClusterController.new(params)
-
-            r.get 'brokers' do
-              controller.brokers
-            end
-
-            r.get 'replication' do
-              controller.replication
-            end
-
-            r.redirect root_path('cluster/brokers')
-          end
-
-          r.on 'topics' do
-            raise Errors::Ui::ProOnlyError
-          end
-
-          r.on 'errors' do
-            controller = Controllers::ErrorsController.new(params)
-
-            r.get Integer do |offset|
-              controller.show(offset)
-            end
-
-            r.get do
-              controller.index
-            end
-          end
-
-          r.get 'status' do
-            controller = Controllers::StatusController.new(params)
-            controller.show
-          end
-
-          r.get 'ux' do
-            controller = Controllers::UxController.new(params)
-            controller.show
-          end
+          SUB_ROUTES.each { |sub_route| sub_route.bind(self, r) }
 
-          r.get 'support' do
-            controller = Controllers::SupportController.new(params)
-            controller.show
-          end
+          nil
         end
       end
     end

data/lib/karafka/web/ui/base.rb

@@ -40,6 +40,7 @@ module Karafka
         plugin :capture_erb
         plugin :content_for
         plugin :inject_erb
+        plugin :all_verbs
 
         # Based on
         # https://github.com/sidekiq/sidekiq/blob/ae6ca119/lib/sidekiq/web/application.rb#L8
@@ -70,7 +71,16 @@ module Karafka
           # Map redirect flashes (if any) to Roda flash messages
           result.flashes.each { |key, value| flash[key] = value }
 
-          response.redirect result.back? ? request.referer : root_path(result.path)
+          path = case result.path
+                 when :back
+                   session[:current_path]
+                 when :previous
+                   session[:previous_path]
+                 else
+                   root_path(result.path)
+                 end
+
+          response.redirect path || root_path
         end
 
         handle_block_result Controllers::Responses::File do |result|
@@ -84,7 +94,8 @@ module Karafka
           ::Rdkafka::RdkafkaError,
           Errors::Ui::NotFoundError,
           Errors::Ui::ProOnlyError,
-          Errors::Ui::ForbiddenError
+          Errors::Ui::ForbiddenError,
+          Errors::Ui::IncompatibleSchemaError
         ] do |e|
           @error = true
 
@@ -95,6 +106,9 @@ module Karafka
           when Errors::Ui::ForbiddenError
             response.status = 403
             view 'shared/exceptions/not_allowed'
+          when Errors::Ui::IncompatibleSchemaError
+            response.status = 422
+            view 'shared/exceptions/incompatible_schema'
           else
             response.status = 404
             view 'shared/exceptions/not_found'
@@ -109,9 +123,11 @@ module Karafka
 
         before do
           check_csrf!
+          store_paths_history(request, session)
         end
 
         plugin :class_matchers
+        plugin :symbol_matchers
 
         # Time matcher with optional hours, minutes and seconds
         TIME_MATCHER = %r{(\d{4}-\d{2}-\d{2}/?(\d{2})?(:\d{2})?(:\d{2})?)}
@@ -122,12 +138,22 @@ module Karafka
         # @note In case the date-time is invalid, raise and render 404
         # @note The time component is optional as `Time#parse` will fallback to lowest time
         #   available, so we can build only date based lookups
-        class_matcher(Time, TIME_MATCHER) do |datetime|
-          [Time.parse(datetime)]
+        class_matcher(Time, TIME_MATCHER) do |*datetime|
+          [Time.parse(datetime[0])]
         rescue ArgumentError
           raise Errors::Ui::NotFoundError
         end
 
+        # Partitions ids cannot be bigger than 32 bit C int. We use this matcher to ensure we
+        # only support that big partition numbers. Otherwise librdkafka would crash.
+        symbol_matcher :partition_id, /(\d{1,14})/ do |value|
+          int_value = value.to_i
+
+          raise Errors::Ui::NotFoundError unless int_value.between?(0, 2_147_483_647)
+
+          [int_value]
+        end
+
         # Allows us to build current path with additional params + it merges existing params into
         # the query data. Query data takes priority over request params.
         # @param query_data [Hash] query params we want to add to the current path
@@ -145,6 +171,14 @@ module Karafka
           [request.path, query_string].compact.join('?')
         end
 
+        # Builds a consumer instance with all needed details
+        # @param consumer_class [Class]
+        def build(consumer_class)
+          Controllers::Requests::ExecutionWrapper.new(
+            consumer_class.new(params, session)
+          )
+        end
+
         # Sets appropriate template variables based on the response object and renders the
         # expected view
         # @param response [Karafka::Web::Ui::Controllers::Responses::Data] response data object
@@ -162,6 +196,31 @@ module Karafka
         def params
           Controllers::Requests::Params.new(request.params)
         end
+
+        private
+
+        # Stores history about visited paths. Useful for redirecting users back when needed.
+        # @param request [Karafka::Web::Ui::App::RodaRequest]
+        # @param session [Object] session object (Rails or Rack)
+        def store_paths_history(request, session)
+          # Code below tracks previous paths so we can use it to redirect users back
+          return unless request.get?
+          return unless request.env['HTTP_ACCEPT']&.include?('text/html')
+
+          requested_path = request.path
+
+          if session[:current_path].nil?
+            session[:current_path] = requested_path
+
+            return
+          end
+
+          return if request.path == session[:current_path]
+
+          # When navigating to a different page
+          session[:previous_path] = session[:current_path]
+          session[:current_path] = requested_path
+        end
       end
     end
   end

data/lib/karafka/web/ui/controllers/base_controller.rb

@@ -8,6 +8,9 @@ module Karafka
         # Base controller from which all the controllers should inherit.
         class BaseController
           include Web::Ui::Lib::Paginations
+          include Requests::Hookable
+
+          attr_reader :params, :session
 
           # Alias for easier referencing
           Models = Web::Ui::Models
@@ -19,9 +22,31 @@ module Karafka
 
           self.sortable_attributes = []
 
+          # Detect that the state of the cache has changed
+          before do
+            cache.clear_if_needed(
+              session[:cache_hash],
+              session[:cache_timestamp].to_i
+            )
+          end
+
+          after do
+            next unless cache.exist?
+
+            session[:cache_hash] = cache.hash
+            session[:cache_timestamp] = cache.timestamp.to_i
+          end
+
           # @param params [Karafka::Web::Ui::Controllers::Requests::Params] request parameters
-          def initialize(params)
+          # @param session [Request::Session] request session (Rails or other framework)
+          def initialize(params, session)
             @params = params
+            @session = session
+          end
+
+          # @return [Karafka::Web::Ui::Lib::Cache] per-process cache instance
+          def cache
+            Karafka::Web.config.ui.cache
           end
 
           private
@@ -50,6 +75,9 @@ module Karafka
 
             attributes[:breadcrums_scope] = scope
 
+            @current_action_name = action.to_sym
+            @current_controller_name = base.join('-')
+
             instance_variables.each do |iv|
               next if iv.to_s.start_with?('@_')
               next if iv.to_s.start_with?('@params')
@@ -72,6 +100,20 @@ module Karafka
             Responses::Redirect.new(path, flashes)
           end
 
+          # Wraps the provided arguments inside a message with a `<strong>` tag to simplify flash
+          # messages building.
+          #
+          # @param message [String] message with `?` to be replaced.
+          # @param args [Array<Object>] arguments to use to replace `?` with strong
+          # @return [String] formatted string
+          def format_flash(message, *args)
+            args.each do |arg|
+              message = message.sub('?', "<strong>#{arg}</strong>")
+            end
+
+            message
+          end
+
           # Builds a file response object that will be used as a base to dispatch the file
           #
           # @param content [String] Payload we want to dispatch as a file

data/lib/karafka/web/ui/controllers/cluster_controller.rb

@@ -17,6 +17,9 @@ module Karafka
             in_sync_replica_brokers
           ].freeze
 
+          # Cluster state should always be fresh and not from cache
+          before { cache.clear }
+
           # Lists available brokers in the cluster
           def brokers
             @brokers = refine(cluster_info.brokers)
@@ -50,9 +53,9 @@ module Karafka
 
           private
 
-          # Make sure, that for the cluster view we always get the most recent cluster state
+          # @return [Array] whole cluster info
           def cluster_info
-            @cluster_info ||= Models::ClusterInfo.fetch(cached: false)
+            @cluster_info ||= Models::ClusterInfo.fetch
           end
 
           # @param cluster_info [Rdkafka::Metadata] cluster metadata

data/lib/karafka/web/ui/controllers/errors_controller.rb

@@ -49,7 +49,7 @@ module Karafka
 
           # @return [String] errors topic
           def errors_topic
-            ::Karafka::Web.config.topics.errors
+            ::Karafka::Web.config.topics.errors.name
           end
         end
       end

data/lib/karafka/web/ui/controllers/requests/execution_wrapper.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Controllers
+        module Requests
+          # @note This class is used internally by the Web UI to wrap controllers
+          #   and inject execution hooks around any method call (before/after filters).
+          #
+          # ExecutionWrapper delegates all method calls to the provided controller instance.
+          # Before and after each invocation, it runs the controller's registered hooks.
+          #
+          # This allows for cleaner separation of concerns and reusable hook logic.
+          #
+          # @example
+          #   controller = SomeController.new
+          #   wrapper = ExecutionWrapper.new(controller)
+          #   wrapper.call # will run before(:call), call, then after(:call)
+          class ExecutionWrapper
+            # @param controller [Object] a controller instance responding to method calls
+            def initialize(controller)
+              @controller = controller
+            end
+
+            # Delegates any method call to the controller and wraps it with before/after hooks
+            #
+            # @param method_name [Symbol] the name of the method being called
+            # @param args [Array] arguments passed to the method
+            # @param block [Proc] optional block passed to the method
+            # @return [Object] the result of the delegated controller method for Roda to operate on
+            def method_missing(method_name, *args, &block)
+              @controller.run_before_hooks(method_name)
+              result = @controller.public_send(method_name, *args, &block)
+              @controller.run_after_hooks(method_name)
+              result
+            end
+
+            # Properly supports respond_to? checks by delegating to the controller
+            #
+            # @param method_name [Symbol] the method name to check
+            # @param include_private [Boolean] whether to include private methods
+            # @return [Boolean] true if controller responds to the method
+            def respond_to_missing?(method_name, include_private = false)
+              @controller.respond_to?(method_name, include_private)
+            end
+          end
+        end
+      end
+    end
+  end
+end