karafka-web 0.10.4 → 0.11.0.beta2

data/lib/karafka/web/processing/consumers/aggregators/state.rb

@@ -36,7 +36,7 @@ module Karafka
             def add(report, offset)
               super(report)
               increment_total_counters(report)
-              update_process_state(report, offset)
+              add_state(report, offset)
               # We always evict after counters updates because we want to use expired (stopped)
               # data for counters as it was valid previously. This can happen only when web consumer
               # had a lag and is catching up.
@@ -46,6 +46,19 @@ module Karafka
               refresh_current_stats
             end
 
+            # Registers or updates the given process state based on the report
+            #
+            # @param report [Hash]
+            # @param offset [Integer]
+            def add_state(report, offset)
+              process_id = report[:process][:id]
+
+              state[:processes][process_id] = {
+                dispatched_at: report[:dispatched_at],
+                offset: offset
+              }
+            end
+
             # @return [Array<Hash, Float>] aggregated current stats value and time from which this
             #   aggregation comes from
             #
@@ -84,19 +97,6 @@ module Karafka
               end
             end
 
-            # Registers or updates the given process state based on the report
-            #
-            # @param report [Hash]
-            # @param offset [Integer]
-            def update_process_state(report, offset)
-              process_id = report[:process][:id]
-
-              state[:processes][process_id] = {
-                dispatched_at: report[:dispatched_at],
-                offset: offset
-              }
-            end
-
             # Evicts expired processes from the current state
             # We consider processes dead if they do not report often enough
             # @note We do not evict based on states (stopped), because we want to report the

data/lib/karafka/web/processing/consumers/metrics.rb

@@ -12,7 +12,7 @@ module Karafka
             # @return [Hash] latest (current) aggregated metrics state
             def current!
               metrics_message = ::Karafka::Admin.read_topic(
-                Karafka::Web.config.topics.consumers.metrics,
+                Karafka::Web.config.topics.consumers.metrics.name,
                 0,
                 # We need to take more in case there would be transactions running.
                 # In theory we could take two but this compensates for any involuntary

data/lib/karafka/web/processing/consumers/state.rb

@@ -12,7 +12,7 @@ module Karafka
             # @return [Hash] last (current) aggregated processes state
             def current!
               state_message = ::Karafka::Admin.read_topic(
-                Karafka::Web.config.topics.consumers.states,
+                Karafka::Web.config.topics.consumers.states.name,
                 0,
                 # We need to take more in case there would be transactions running.
                 # In theory we could take two but this compensates for any involuntary

data/lib/karafka/web/processing/publisher.rb

@@ -36,17 +36,17 @@ module Karafka
           def prepare_data(consumers_state, consumers_metrics)
             [
               {
-                topic: Karafka::Web.config.topics.consumers.states,
+                topic: Karafka::Web.config.topics.consumers.states.name,
                 payload: Zlib::Deflate.deflate(consumers_state.to_json),
                 # This will ensure that the consumer states are compacted
-                key: Karafka::Web.config.topics.consumers.states,
+                key: Karafka::Web.config.topics.consumers.states.name,
                 partition: 0,
                 headers: { 'zlib' => 'true' }
               },
               {
-                topic: Karafka::Web.config.topics.consumers.metrics,
+                topic: Karafka::Web.config.topics.consumers.metrics.name,
                 payload: Zlib::Deflate.deflate(consumers_metrics.to_json),
-                key: Karafka::Web.config.topics.consumers.metrics,
+                key: Karafka::Web.config.topics.consumers.metrics.name,
                 partition: 0,
                 headers: { 'zlib' => 'true' }
               }

data/lib/karafka/web/tracking/consumers/contracts/partition.rb

@@ -28,6 +28,7 @@ module Karafka
             required(:ls_offset) { |val| val.is_a?(Integer) }
             required(:ls_offset_d) { |val| val.is_a?(Integer) }
             required(:ls_offset_fd) { |val| val.is_a?(Integer) && val >= 0 }
+            required(:transactional) { |val| [true, false].include?(val) }
           end
         end
       end

data/lib/karafka/web/tracking/consumers/listeners/pausing.rb

@@ -8,10 +8,10 @@ module Karafka
           # Tracks pausing and un-pausing of topics partitions for both user requested and
           # automatic events.
           class Pausing < Base
-            # Indicate pause
+            # Tracks the pause start
             #
             # @param event [Karafka::Core::Monitoring::Event]
-            def on_consumer_consuming_pause(event)
+            def on_client_pause(event)
               track do |sampler|
                 sampler.pauses[pause_id(event)] = {
                   timeout: event[:timeout],

data/lib/karafka/web/tracking/consumers/listeners/transactions.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Consumers
+        module Listeners
+          # Tracks data related to transactions
+          # seek offsets are needed because when consumer offsets are committed in transactions,
+          # librdkafka does not publish the lags in a regular way (they are set to -1) and we need
+          # to compute them via enrichment of information.
+          class Transactions < Base
+            # Tracking of things needed to support transactional consumers post successful
+            # transaction.
+            #
+            # @param event [Karafka::Core::Monitoring::Event]
+            def on_consumer_consuming_transaction(event)
+              consumer = event[:caller]
+              sg_id = consumer.topic.subscription_group.id
+              topic_name = consumer.topic.name
+              # We store it as a string because librdkafka also does that and its easier to align
+              # without casting it later
+              partition_id = consumer.partition
+
+              track do |sampler|
+                break unless sampler.subscription_groups.key?(sg_id)
+
+                seek_offset = consumer.coordinator.seek_offset
+
+                break if seek_offset.nil?
+
+                topics_scope = sampler.subscription_groups[sg_id][:topics]
+                p_scope = topics_scope[topic_name][partition_id]
+
+                p_scope[:transactional] = true
+                p_scope[:seek_offset] = seek_offset
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/consumers/reporter.rb

@@ -63,7 +63,7 @@ module Karafka
               # Report consumers statuses
               messages = [
                 {
-                  topic: ::Karafka::Web.config.topics.consumers.reports,
+                  topic: ::Karafka::Web.config.topics.consumers.reports.name,
                   payload: Zlib::Deflate.deflate(report.to_json),
                   key: process_id,
                   partition: 0,
@@ -76,7 +76,7 @@ module Karafka
                 @error_contract.validate!(error)
 
                 {
-                  topic: Karafka::Web.config.topics.errors,
+                  topic: Karafka::Web.config.topics.errors.name,
                   payload: Zlib::Deflate.deflate(error.to_json),
                   # Always dispatch errors from the same process to the same partition
                   key: process_id,

data/lib/karafka/web/tracking/consumers/sampler.rb

@@ -15,7 +15,7 @@ module Karafka
           # Current schema version
           # This is used for detecting incompatible changes and not using outdated data during
           # upgrades
-          SCHEMA_VERSION = '1.4.0'
+          SCHEMA_VERSION = '1.4.1'
 
           # Counters that count events occurrences during the given window
           COUNTERS_BASE = {
@@ -51,7 +51,17 @@ module Karafka
             @subscription_groups = Hash.new do |h, sg_id|
               h[sg_id] = {
                 id: sg_id,
-                polled_at: monotonic_now
+                polled_at: monotonic_now,
+                topics: Hash.new do |h1, topic|
+                  h1[topic] = Hash.new do |h2, partition|
+                    # We track those details in case we need to fill statistical gaps for
+                    # transactional consumers
+                    h2[partition] = {
+                      seek_offset: -1,
+                      transactional: false
+                    }
+                  end
+                end
               }
             end
 
@@ -223,11 +233,9 @@ module Karafka
           def memory_size
             @memory_size ||= case RUBY_PLATFORM
                              when /linux/
-                               @shell
-                             .call('grep MemTotal /proc/meminfo')
-                             .match(/(\d+)/)
-                             .to_s
-                             .to_i
+                               mem_info = File.read('/proc/meminfo')
+                               mem_total_line = mem_info.match(/MemTotal:\s*(?<total>\d+)/)
+                               mem_total_line['total'].to_i
                              when /darwin|bsd/
                                @shell
                              .call('sysctl -a')
@@ -245,7 +253,13 @@ module Karafka
           # @return [Array<Float>] load averages for last 1, 5 and 15 minutes
           def cpu_usage
             case RUBY_PLATFORM
-            when /darwin|bsd|linux/
+            when /linux/
+              File
+                .read('/proc/loadavg')
+                .split(' ')
+                .first(3)
+                .map(&:to_f)
+            when /darwin|bsd/
               @shell
                 .call('w | head -1')
                 .strip
@@ -280,10 +294,21 @@ module Karafka
           def memory_threads_ps
             @memory_threads_ps = case RUBY_PLATFORM
                                  when /linux/
-                                   @shell
-                                 .call('ps -A -o rss=,thcount=,pid=')
-                                 .split("\n")
-                                 .map { |row| row.strip.split(' ').map(&:to_i) }
+                                   page_size = Karafka::Web::Tracking::Helpers::Sysconf.page_size
+                                   status_file = "/proc/#{::Process.pid}/status"
+
+                                   pid = status_file.match(%r{/proc/(\d+)/status})[1]
+
+                                   # Extract thread count from /proc/<pid>/status
+                                   thcount = File.read(status_file)[/^Threads:\s+(\d+)/, 1].to_i
+
+                                   # Extract RSS from /proc/<pid>/statm (second field)
+                                   statm_file = "/proc/#{pid}/statm"
+                                   rss_pages = File.read(statm_file).split[1].to_i rescue 0
+                                   # page size is retrieved from Sysconf
+                                   rss_kb = (rss_pages * page_size) / 1024
+
+                                   [[rss_kb, thcount, pid.to_i]]
                                  # thcount is not available on macos ps
                                  # because of that we inject 0 as threads count similar to how
                                  # we do on windows
@@ -308,9 +333,51 @@ module Karafka
                   # This should be always available, since the subscription group polled at time
                   # is first initialized before we start polling, there should be no case where
                   # we have statistics about a given subscription group but we do not have the
-                  # last polling time
-                  polled_at = subscription_groups.fetch(sg_id).fetch(:polled_at)
+                  # sg reference
+                  sg_tracking = subscription_groups.fetch(sg_id)
+
+                  polled_at = sg_tracking.fetch(:polled_at)
                   sg_details[:state][:poll_age] = (monotonic_now - polled_at).round(2)
+
+                  sg_details[:topics].each do |topic_name, topic_details|
+                    topic_details[:partitions].each do |partition_id, partition_details|
+                      # Always assume non-transactional as default. Will be overwritten by the
+                      # consumer level details if collected
+                      partition_details[:transactional] ||= false
+
+                      # If we have stored offset or stored lag, it means it's not a transactional
+                      # consumer at all so we can skip enrichment
+                      next if partition_details[:lag_stored].positive?
+                      next if partition_details[:stored_offset].positive?
+                      next unless sg_tracking[:topics].key?(topic_name)
+                      next unless sg_tracking[:topics][topic_name].key?(partition_id)
+
+                      k_partition_details = sg_tracking[:topics][topic_name][partition_id]
+
+                      # If seek offset was not yey set, nothing to enrich
+                      next unless k_partition_details[:seek_offset].positive?
+
+                      partition_details[:transactional] = k_partition_details[:transactional]
+
+                      # Seek offset is always +1 from the last stored in Karafka
+                      seek_offset = k_partition_details[:seek_offset]
+                      stored_offset = seek_offset - 1
+
+                      # In case of transactions we have to compute the lag ourselves
+                      # -1 because ls offset (or high watermark) is last + 1
+                      lag = partition_details[:ls_offset] - seek_offset
+                      # This can happen if ls_offset is refreshed slower than our stored offset
+                      # fetching from Karafka transactional layer
+                      lag = 0 if lag.negative?
+
+                      partition_details[:lag] = lag
+                      partition_details[:lag_d] = 0
+                      partition_details[:lag_stored] = lag
+                      partition_details[:lag_stored_d] = 0
+                      partition_details[:stored_offset] = stored_offset
+                      partition_details[:committed_offset] = stored_offset
+                    end
+                  end
                 end
               end
             end

data/lib/karafka/web/tracking/helpers/sysconf.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      # Namespace for tracking related helpers
+      module Helpers
+        # Namespace for unix-based helper methods used to fetch OS details
+        module Sysconf
+          extend FFI::Library
+
+          case RUBY_PLATFORM
+          when /linux/
+            ffi_lib 'libc.so.6' # Standard C library on Linux
+            SC_PAGESIZE = 30 # _SC_PAGESIZE constant
+          when /darwin/
+            ffi_lib 'libSystem.B.dylib' # Standard C library on macOS
+            SC_PAGESIZE = 29 # _SC_PAGESIZE constant
+          end
+
+          attach_function :sysconf, [:int], :long
+
+          class << self
+            # @return [Integer]
+            def page_size
+              sysconf(SC_PAGESIZE)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/producers/reporter.rb

@@ -33,7 +33,7 @@ module Karafka
                 @error_contract.validate!(error)
 
                 {
-                  topic: Karafka::Web.config.topics.errors,
+                  topic: Karafka::Web.config.topics.errors.name,
                   payload: error.to_json,
                   # Always dispatch errors from the same process to the same partition
                   key: error[:process][:id]

data/lib/karafka/web/ui/app.rb

@@ -11,122 +11,29 @@ module Karafka
 
         instance_exec(&CONTEXT_DETAILS)
 
+        # Sub-routes for given pieces of the Web UI
+        SUB_ROUTES = [
+          Routes::Assets,
+          Routes::Dashboard,
+          Routes::Consumers,
+          Routes::ProOnly,
+          Routes::Jobs,
+          Routes::Routing,
+          Routes::Cluster,
+          Routes::Errors,
+          Routes::Status,
+          Routes::Support,
+          Routes::Ux
+        ].freeze
+
+        private_constant :SUB_ROUTES
+
         route do |r|
           r.root { r.redirect root_path('dashboard') }
 
-          # Serve current version specific assets to prevent users from fetching old assets
-          # after upgrade
-          r.on 'assets', Karafka::Web::VERSION do
-            r.public
-          end
-
-          r.get 'dashboard' do
-            @breadcrumbs = false
-            controller = Controllers::DashboardController.new(params)
-            controller.index
-          end
-
-          r.on 'consumers' do
-            %w[
-              performance
-              controls
-              commands
-            ].each do |path|
-              r.get path do |_process_id|
-                raise Errors::Ui::ProOnlyError
-              end
-            end
-
-            r.get String, 'subscriptions' do |_process_id|
-              raise Errors::Ui::ProOnlyError
-            end
-
-            r.get do
-              controller = Controllers::ConsumersController.new(params)
-              controller.index
-            end
-          end
-
-          %w[
-            health
-            explorer
-            dlq
-          ].each do |route|
-            r.get route, [String, true], [String, true] do
-              raise Errors::Ui::ProOnlyError
-            end
-          end
-
-          r.on 'jobs' do
-            controller = Controllers::JobsController.new(params)
-
-            r.get 'running' do
-              controller.running
-            end
-
-            r.get 'pending' do
-              controller.pending
-            end
-
-            r.redirect root_path('jobs/running')
-          end
-
-          r.on 'routing' do
-            controller = Controllers::RoutingController.new(params)
-
-            r.get String do |topic_id|
-              controller.show(topic_id)
-            end
-
-            r.get do
-              controller.index
-            end
-          end
-
-          r.on 'cluster' do
-            controller = Controllers::ClusterController.new(params)
-
-            r.get 'brokers' do
-              controller.brokers
-            end
-
-            r.get 'replication' do
-              controller.replication
-            end
-
-            r.redirect root_path('cluster/brokers')
-          end
-
-          r.on 'topics' do
-            raise Errors::Ui::ProOnlyError
-          end
-
-          r.on 'errors' do
-            controller = Controllers::ErrorsController.new(params)
-
-            r.get Integer do |offset|
-              controller.show(offset)
-            end
-
-            r.get do
-              controller.index
-            end
-          end
-
-          r.get 'status' do
-            controller = Controllers::StatusController.new(params)
-            controller.show
-          end
-
-          r.get 'ux' do
-            controller = Controllers::UxController.new(params)
-            controller.show
-          end
+          SUB_ROUTES.each { |sub_route| sub_route.bind(self, r) }
 
-          r.get 'support' do
-            controller = Controllers::SupportController.new(params)
-            controller.show
-          end
+          nil
         end
       end
     end

data/lib/karafka/web/ui/base.rb

@@ -40,6 +40,7 @@ module Karafka
         plugin :capture_erb
         plugin :content_for
         plugin :inject_erb
+        plugin :all_verbs
 
         # Based on
         # https://github.com/sidekiq/sidekiq/blob/ae6ca119/lib/sidekiq/web/application.rb#L8
@@ -70,7 +71,16 @@ module Karafka
           # Map redirect flashes (if any) to Roda flash messages
           result.flashes.each { |key, value| flash[key] = value }
 
-          response.redirect result.back? ? request.referer : root_path(result.path)
+          path = case result.path
+                 when :back
+                   session[:current_path]
+                 when :previous
+                   session[:previous_path]
+                 else
+                   root_path(result.path)
+                 end
+
+          response.redirect path || root_path
         end
 
         handle_block_result Controllers::Responses::File do |result|
@@ -109,9 +119,11 @@ module Karafka
 
         before do
           check_csrf!
+          store_paths_history(request, session)
         end
 
         plugin :class_matchers
+        plugin :symbol_matchers
 
         # Time matcher with optional hours, minutes and seconds
         TIME_MATCHER = %r{(\d{4}-\d{2}-\d{2}/?(\d{2})?(:\d{2})?(:\d{2})?)}
@@ -122,12 +134,22 @@ module Karafka
         # @note In case the date-time is invalid, raise and render 404
         # @note The time component is optional as `Time#parse` will fallback to lowest time
         #   available, so we can build only date based lookups
-        class_matcher(Time, TIME_MATCHER) do |datetime|
-          [Time.parse(datetime)]
+        class_matcher(Time, TIME_MATCHER) do |*datetime|
+          [Time.parse(datetime[0])]
         rescue ArgumentError
           raise Errors::Ui::NotFoundError
         end
 
+        # Partitions ids cannot be bigger than 32 bit C int. We use this matcher to ensure we
+        # only support that big partition numbers. Otherwise librdkafka would crash.
+        symbol_matcher :partition_id, /(\d{1,14})/ do |value|
+          int_value = value.to_i
+
+          raise Errors::Ui::NotFoundError unless int_value.between?(0, 2_147_483_647)
+
+          [int_value]
+        end
+
         # Allows us to build current path with additional params + it merges existing params into
         # the query data. Query data takes priority over request params.
         # @param query_data [Hash] query params we want to add to the current path
@@ -145,6 +167,14 @@ module Karafka
           [request.path, query_string].compact.join('?')
         end
 
+        # Builds a consumer instance with all needed details
+        # @param consumer_class [Class]
+        def build(consumer_class)
+          Controllers::Requests::ExecutionWrapper.new(
+            consumer_class.new(params, session)
+          )
+        end
+
         # Sets appropriate template variables based on the response object and renders the
         # expected view
         # @param response [Karafka::Web::Ui::Controllers::Responses::Data] response data object
@@ -162,6 +192,31 @@ module Karafka
         def params
           Controllers::Requests::Params.new(request.params)
         end
+
+        private
+
+        # Stores history about visited paths. Useful for redirecting users back when needed.
+        # @param request [Karafka::Web::Ui::App::RodaRequest]
+        # @param session [Object] session object (Rails or Rack)
+        def store_paths_history(request, session)
+          # Code below tracks previous paths so we can use it to redirect users back
+          return unless request.get?
+          return unless request.env['HTTP_ACCEPT']&.include?('text/html')
+
+          requested_path = request.path
+
+          if session[:current_path].nil?
+            session[:current_path] = requested_path
+
+            return
+          end
+
+          return if request.path == session[:current_path]
+
+          # When navigating to a different page
+          session[:previous_path] = session[:current_path]
+          session[:current_path] = requested_path
+        end
       end
     end
   end

data/lib/karafka/web/ui/controllers/base_controller.rb

@@ -8,6 +8,9 @@ module Karafka
         # Base controller from which all the controllers should inherit.
         class BaseController
           include Web::Ui::Lib::Paginations
+          include Requests::Hookable
+
+          attr_reader :params, :session
 
           # Alias for easier referencing
           Models = Web::Ui::Models
@@ -19,9 +22,31 @@ module Karafka
 
           self.sortable_attributes = []
 
+          # Detect that the state of the cache has changed
+          before do
+            cache.clear_if_needed(
+              session[:cache_hash],
+              session[:cache_timestamp].to_i
+            )
+          end
+
+          after do
+            next unless cache.exist?
+
+            session[:cache_hash] = cache.hash
+            session[:cache_timestamp] = cache.timestamp.to_i
+          end
+
           # @param params [Karafka::Web::Ui::Controllers::Requests::Params] request parameters
-          def initialize(params)
+          # @param session [Request::Session] request session (Rails or other framework)
+          def initialize(params, session)
             @params = params
+            @session = session
+          end
+
+          # @return [Karafka::Web::Ui::Lib::Cache] per-process cache instance
+          def cache
+            Karafka::Web.config.ui.cache
           end
 
           private
@@ -50,6 +75,9 @@ module Karafka
 
             attributes[:breadcrums_scope] = scope
 
+            @current_action_name = action.to_sym
+            @current_controller_name = base.join('-')
+
             instance_variables.each do |iv|
               next if iv.to_s.start_with?('@_')
               next if iv.to_s.start_with?('@params')
@@ -72,6 +100,20 @@ module Karafka
             Responses::Redirect.new(path, flashes)
           end
 
+          # Wraps the provided arguments inside a message with a `<strong>` tag to simplify flash
+          # messages building.
+          #
+          # @param message [String] message with `?` to be replaced.
+          # @param args [Array<Object>] arguments to use to replace `?` with strong
+          # @return [String] formatted string
+          def format_flash(message, *args)
+            args.each do |arg|
+              message = message.sub('?', "<strong>#{arg}</strong>")
+            end
+
+            message
+          end
+
           # Builds a file response object that will be used as a base to dispatch the file
           #
           # @param content [String] Payload we want to dispatch as a file