karafka-web 0.6.3 → 0.7.0

data/lib/karafka/web/tracking/consumers/contracts/report.rb

@@ -9,7 +9,7 @@ module Karafka
           #
           # Any outgoing reporting needs to match this format for it to work with the statuses
           # consumer.
-          class Report < Tracking::Contracts::Base
+          class Report < Web::Contracts::Base
             configure
 
             required(:schema_version) { |val| val.is_a?(String) && !val.empty? }
@@ -21,12 +21,14 @@ module Karafka
             nested(:process) do
               required(:started_at) { |val| val.is_a?(Numeric) && val.positive? }
               required(:name) { |val| val.is_a?(String) && val.count(':') >= 2 }
+              required(:cpus) { |val| val.is_a?(Integer) && val >= 1 }
               required(:memory_usage) { |val| val.is_a?(Integer) && val >= 0 }
               required(:memory_total_usage) { |val| val.is_a?(Integer) && val >= 0 }
               required(:memory_size) { |val| val.is_a?(Integer) && val >= 0 }
               required(:status) { |val| ::Karafka::Status::STATES.key?(val.to_s.to_sym) }
+              required(:threads) { |val| val.is_a?(Integer) && val >= 0 }
               required(:listeners) { |val| val.is_a?(Integer) && val >= 0 }
-              required(:concurrency) { |val| val.is_a?(Integer) && val.positive? }
+              required(:workers) { |val| val.is_a?(Integer) && val.positive? }
               required(:tags) { |val| val.is_a?(Karafka::Core::Taggable::Tags) }
 
               required(:cpu_usage) do |val|
@@ -53,11 +55,11 @@ module Karafka
               required(:utilization) { |val| val.is_a?(Numeric) && val >= 0 }
 
               nested(:total) do
-                required(:batches) { |val| val.is_a?(Numeric) && val >= 0 }
-                required(:messages) { |val| val.is_a?(Numeric) && val >= 0 }
-                required(:errors) { |val| val.is_a?(Numeric) && val >= 0 }
-                required(:retries) { |val| val.is_a?(Numeric) && val >= 0 }
-                required(:dead) { |val| val.is_a?(Numeric) && val >= 0 }
+                required(:batches) { |val| val.is_a?(Integer) && val >= 0 }
+                required(:messages) { |val| val.is_a?(Integer) && val >= 0 }
+                required(:errors) { |val| val.is_a?(Integer) && val >= 0 }
+                required(:retries) { |val| val.is_a?(Integer) && val >= 0 }
+                required(:dead) { |val| val.is_a?(Integer) && val >= 0 }
               end
             end
 
@@ -73,7 +75,7 @@ module Karafka
               cg_contract = ConsumerGroup.new
 
               # Consumer group id (key) is irrelevant because it is also in the details
-              data.fetch(:consumer_groups).each do |_, details|
+              data.fetch(:consumer_groups).each_value do |details|
                 cg_contract.validate!(details)
               end
 

data/lib/karafka/web/tracking/consumers/contracts/subscription_group.rb

@@ -7,7 +7,7 @@ module Karafka
         module Contracts
           # Expected data for each subscription group
           # It's mostly about topics details
-          class SubscriptionGroup < Tracking::Contracts::Base
+          class SubscriptionGroup < Web::Contracts::Base
             configure
 
             required(:id) { |val| val.is_a?(String) && !val.empty? }
@@ -19,7 +19,7 @@ module Karafka
 
               topic_contract = Topic.new
 
-              data.fetch(:topics).each do |_topic_name, details|
+              data.fetch(:topics).each_value do |details|
                 topic_contract.validate!(details)
               end
 

data/lib/karafka/web/tracking/consumers/contracts/topic.rb

@@ -6,7 +6,7 @@ module Karafka
       module Consumers
         module Contracts
           # Expected topic information that needs to go out
-          class Topic < Tracking::Contracts::Base
+          class Topic < Web::Contracts::Base
             configure
 
             required(:name) { |val| val.is_a?(String) && !val.empty? }
@@ -17,7 +17,7 @@ module Karafka
 
               partition_contract = Partition.new
 
-              data.fetch(:partitions).each do |_partition_id, details|
+              data.fetch(:partitions).each_value do |details|
                 partition_contract.validate!(details)
               end
 

data/lib/karafka/web/tracking/consumers/listeners/processing.rb

@@ -21,7 +21,7 @@ module Karafka
             # @param event [Karafka::Core::Monitoring::Event]
             def on_consumer_consume(event)
               consumer = event.payload[:caller]
-              messages_count = consumer.messages.count
+              messages_count = consumer.messages.size
               jid = job_id(consumer, 'consume')
               job_details = job_details(consumer, 'consume')
 
@@ -73,7 +73,7 @@ module Karafka
               consumer = event.payload[:caller]
               topic = consumer.topic
               consumer_group_id = topic.consumer_group.id
-              messages_count = consumer.messages.count
+              messages_count = consumer.messages.size
               time = event[:time]
               jid = job_id(consumer, 'consume')
 
@@ -160,6 +160,10 @@ module Karafka
                 processing_lag: consumer.messages.metadata.processing_lag,
                 consumption_lag: consumer.messages.metadata.consumption_lag,
                 committed_offset: consumer.coordinator.seek_offset - 1,
+                # In theory this is redundant because we have first and last offset, but it is
+                # needed because VPs do not have linear count. For VPs first and last offset
+                # will be further away than the total messages count for a particular VP
+                messages: consumer.messages.size,
                 consumer: consumer.class.to_s,
                 consumer_group: consumer.topic.consumer_group.id,
                 type: type,

data/lib/karafka/web/tracking/consumers/listeners/statistics.rb

@@ -104,11 +104,25 @@ module Karafka
             # @return [Hash] extracted partition metrics
             def extract_partition_metrics(pt_stats)
               metrics = pt_stats.slice(
+                'consumer_lag',
+                'consumer_lag_d',
                 'consumer_lag_stored',
                 'consumer_lag_stored_d',
                 'committed_offset',
+                # Can be useful to track the frequency of flushes when there is progress
+                'committed_offset_fd',
                 'stored_offset',
-                'fetch_state'
+                # Can be useful to track the frequency of flushes when there is progress
+                'stored_offset_fd',
+                'fetch_state',
+                'hi_offset',
+                'hi_offset_fd',
+                'lo_offset',
+                'eof_offset',
+                'ls_offset',
+                # Two below can be useful for detection of hanging transactions
+                'ls_offset_d',
+                'ls_offset_fd'
               )
 
               # Rename as we do not need `consumer_` prefix

data/lib/karafka/web/tracking/consumers/reporter.rb

@@ -32,6 +32,14 @@ module Karafka
           # @param forced [Boolean] should we report bypassing the time frequency or should we
           #   report only in case we would not send the report for long enough time.
           def report(forced: false)
+            # Do not even mutex if not needed
+            return unless report?(forced)
+
+            # We run this sampling before the mutex so sampling does not stop things in case
+            # other threads would need this mutex. This can take up to 25ms and we do not want to
+            # block during this time
+            sampler.sample
+
             MUTEX.synchronize do
               # Start background thread only when needed
               # This prevents us from starting it too early or for non-consumer processes where
@@ -52,9 +60,10 @@ module Karafka
               messages = [
                 {
                   topic: ::Karafka::Web.config.topics.consumers.reports,
-                  payload: report.to_json,
+                  payload: Zlib::Deflate.deflate(report.to_json),
                   key: process_name,
-                  partition: 0
+                  partition: 0,
+                  headers: { 'zlib' => 'true' }
                 }
               ]
 
@@ -64,14 +73,13 @@ module Karafka
 
                 {
                   topic: Karafka::Web.config.topics.errors,
-                  payload: error.to_json,
+                  payload: Zlib::Deflate.deflate(error.to_json),
                   # Always dispatch errors from the same process to the same partition
-                  key: process_name
+                  key: process_name,
+                  headers: { 'zlib' => 'true' }
                 }
               end
 
-              return if messages.empty?
-
               produce(messages)
 
               # Clear the sampler so it tracks new state changes without previous once impacting

data/lib/karafka/web/tracking/consumers/sampler.rb

@@ -14,7 +14,7 @@ module Karafka
           # Current schema version
           # This can be used in the future for detecting incompatible changes and writing
           # migrations
-          SCHEMA_VERSION = '1.2.0'
+          SCHEMA_VERSION = '1.2.3'
 
           # 60 seconds window for time tracked window-based metrics
           TIMES_TTL = 60
@@ -22,18 +22,26 @@ module Karafka
           # Times ttl in ms
           TIMES_TTL_MS = TIMES_TTL * 1_000
 
-          private_constant :TIMES_TTL, :TIMES_TTL_MS, :SCHEMA_VERSION
+          # Counters that count events occurrences during the given window
+          COUNTERS_BASE = {
+            # Number of processed batches
+            batches: 0,
+            # Number of processed messages
+            messages: 0,
+            # Number of errors that occurred
+            errors: 0,
+            # Number of retries that occurred
+            retries: 0,
+            # Number of messages considered dead
+            dead: 0
+          }.freeze
+
+          private_constant :TIMES_TTL, :TIMES_TTL_MS, :COUNTERS_BASE
 
           def initialize
             super
 
-            @counters = {
-              batches: 0,
-              messages: 0,
-              errors: 0,
-              retries: 0,
-              dead: 0
-            }
+            @counters = COUNTERS_BASE.dup
             @times = TtlHash.new(TIMES_TTL_MS)
             @consumer_groups = {}
             @errors = []
@@ -41,6 +49,9 @@ module Karafka
             @pauses = Set.new
             @jobs = {}
             @shell = MemoizedShell.new
+            @memory_total_usage = 0
+            @memory_usage = 0
+            @cpu_usage = [-1, -1, -1]
           end
 
           # We cannot report and track the same time, that is why we use mutex here. To make sure
@@ -63,12 +74,13 @@ module Karafka
                 name: process_name,
                 status: ::Karafka::App.config.internal.status.to_s,
                 listeners: listeners,
-                concurrency: concurrency,
-                memory_usage: memory_usage,
-                memory_total_usage: memory_total_usage,
+                workers: workers,
+                memory_usage: @memory_usage,
+                memory_total_usage: @memory_total_usage,
                 memory_size: memory_size,
-                cpu_count: cpu_count,
-                cpu_usage: cpu_usage,
+                cpus: cpus,
+                threads: threads,
+                cpu_usage: @cpu_usage,
                 tags: Karafka::Process.tags
               },
 
@@ -101,6 +113,17 @@ module Karafka
             @errors.clear
           end
 
+          # @note This should run before any mutex, so other threads can continue as those
+          #   operations may invoke shell commands
+          def sample
+            memory_threads_ps
+
+            @memory_usage = memory_usage
+            @memory_total_usage = memory_total_usage
+            @cpu_usage = cpu_usage
+            @threads = threads
+          end
+
           private
 
           # @return [Numeric] % utilization of all the threads. 100% means all the threads are
@@ -115,7 +138,7 @@ module Karafka
 
             # We divide by 1_000 to convert from milliseconds
             # We multiply by 100 to have it in % scale
-            times[:total].sum / 1_000 / concurrency / timefactor * 100
+            times[:total].sum / 1_000 / workers / timefactor * 100
           end
 
           # @return [Integer] number of listeners
@@ -152,20 +175,16 @@ module Karafka
           # @return [Hash] job queue statistics
           def jobs_queue_statistics
             # We return empty stats in case jobs queue is not yet initialized
+            # busy - represents number of jobs that are being executed currently
+            # enqueued - represents number of jobs that are enqueued to be processed
             Karafka::Server.jobs_queue&.statistics || { busy: 0, enqueued: 0 }
           end
 
           # Total memory used in the OS
           def memory_total_usage
-            case RUBY_PLATFORM
-            when /darwin|bsd|linux/
-              @shell
-                .call('ps -A -o rss=')
-                .split("\n")
-                .inject { |a, e| a.to_i + e.strip.to_i }
-            else
-              0
-            end
+            return 0 unless @memory_threads_ps
+
+            @memory_threads_ps.map(&:first).sum
           end
 
           # @return [Integer] total amount of memory
@@ -173,19 +192,19 @@ module Karafka
             @memory_size ||= case RUBY_PLATFORM
                              when /linux/
                                @shell
-                                 .call('grep MemTotal /proc/meminfo')
-                                 .match(/(\d+)/)
-                                 .to_s
-                                 .to_i
+                             .call('grep MemTotal /proc/meminfo')
+                             .match(/(\d+)/)
+                             .to_s
+                             .to_i
                              when /darwin|bsd/
                                @shell
-                                 .call('sysctl -a')
-                                 .split("\n")
-                                 .find { |line| line.start_with?('hw.memsize:') }
-                                 .to_s
-                                 .split(' ')
-                                 .last
-                                 .to_i
+                             .call('sysctl -a')
+                             .split("\n")
+                             .find { |line| line.start_with?('hw.memsize:') }
+                             .to_s
+                             .split(' ')
+                             .last
+                             .to_i
                              else
                                0
                              end
@@ -206,14 +225,36 @@ module Karafka
             end
           end
 
+          # @return [Integer] number of process threads.
+          # @note This returns total number of threads from the OS perspective including native
+          #   extensions threads, etc.
+          def threads
+            return 0 unless @memory_threads_ps
+
+            @memory_threads_ps.find { |row| row.last == ::Process.pid }[1]
+          end
+
           # @return [Integer] CPU count
-          def cpu_count
-            @cpu_count ||= Etc.nprocessors
+          def cpus
+            @cpus ||= Etc.nprocessors
           end
 
           # @return [Integer] number of threads that process work
-          def concurrency
-            @concurrency ||= Karafka::App.config.concurrency
+          def workers
+            @workers ||= Karafka::App.config.concurrency
+          end
+
+          # Loads our ps results into memory so we can extract from them whatever we need
+          def memory_threads_ps
+            @memory_threads_ps = case RUBY_PLATFORM
+                                 when /darwin|bsd|linux/
+                                   @shell
+                                 .call('ps -A -o rss=,thcount,pid')
+                                 .split("\n")
+                                 .map { |row| row.strip.split(' ').map(&:to_i) }
+                                 else
+                                   @memory_threads_ps = false
+                                 end
           end
         end
       end

data/lib/karafka/web/tracking/contracts/error.rb

@@ -3,11 +3,12 @@
 module Karafka
   module Web
     module Tracking
+      # Namespace for all tracking related contracts
       module Contracts
         # Contract for error reporting
         # Since producers and consumers report their errors to the same topic, we need to have
         # a unified contract for both
-        class Error < Base
+        class Error < Web::Contracts::Base
           configure
 
           required(:schema_version) { |val| val.is_a?(String) }

data/lib/karafka/web/ui/app.rb

@@ -12,9 +12,19 @@ module Karafka
         instance_exec(&CONTEXT_DETAILS)
 
         route do |r|
-          r.root { r.redirect root_path('consumers') }
+          r.root { r.redirect root_path('dashboard') }
 
-          @current_page = params.current_page
+          # Serve current version specific assets to prevent users from fetching old assets
+          # after upgrade
+          r.on(:assets, Karafka::Web::VERSION) do
+            r.public
+          end
+
+          r.get 'dashboard' do
+            @breadcrumbs = false
+            controller = Controllers::Dashboard.new(params)
+            controller.index
+          end
 
           r.on 'consumers' do
             r.get String, 'subscriptions' do |_process_id|
@@ -24,7 +34,7 @@ module Karafka
             r.get do
               @breadcrumbs = false
               controller = Controllers::Consumers.new(params)
-              render_response controller.index
+              controller.index
             end
           end
 
@@ -40,41 +50,41 @@ module Karafka
 
           r.get 'jobs' do
             controller = Controllers::Jobs.new(params)
-            render_response controller.index
+            controller.index
           end
 
           r.on 'routing' do
             controller = Controllers::Routing.new(params)
 
             r.get String do |topic_id|
-              render_response controller.show(topic_id)
+              controller.show(topic_id)
             end
 
             r.get do
-              render_response controller.index
+              controller.index
             end
           end
 
           r.get 'cluster' do
             controller = Controllers::Cluster.new(params)
-            render_response controller.index
+            controller.index
           end
 
           r.on 'errors' do
             controller = Controllers::Errors.new(params)
 
             r.get Integer do |offset|
-              render_response controller.show(offset)
+              controller.show(offset)
             end
 
             r.get do
-              render_response controller.index
+              controller.index
             end
           end
 
           r.get 'status' do
             controller = Controllers::Status.new(params)
-            render_response controller.show
+            controller.show
           end
         end
       end

data/lib/karafka/web/ui/base.rb

@@ -5,24 +5,34 @@ module Karafka
     module Ui
       # Base Roda application
       class Base < Roda
+        include Helpers::PathsHelper
         include Helpers::ApplicationHelper
 
         # Details that need to be evaluated in the context of OSS or Pro web UI.
         # If those would be evaluated in the base, they would not be initialized as expected
         CONTEXT_DETAILS = lambda do
           plugin(
-            :static,
-            %w[/javascripts /stylesheets /images],
-            root: Karafka::Web.gem_root.join('lib/karafka/web/ui/public')
+            :public,
+            root: Karafka::Web.gem_root.join('lib/karafka/web/ui/public'),
+            # Cache all static files for the end user for as long as possible
+            # We can do it because we ship per version assets so they invalidate with gem bumps
+            headers: { 'Cache-Control' => 'max-age=604800' }
           )
           plugin :render_each
           plugin :partials
+          # The secret here will be reconfigured after Web UI configuration setup
+          # This is why we assign here a random value as it will have to be changed by the end
+          # user to make the Web UI work.
+          plugin :sessions, key: '_karafka_session', secret: SecureRandom.hex(64)
+          plugin :route_csrf
         end
 
         plugin :render, escape: true, engine: 'erb'
         plugin :run_append_slash
         plugin :error_handler
         plugin :not_found
+        plugin :hooks
+        plugin :flash
         plugin :path
 
         # Based on
@@ -43,15 +53,30 @@ module Karafka
           csp.base_uri "'self'"
         end
 
+        plugin :custom_block_results
+
+        handle_block_result Controllers::Responses::Data do |result|
+          render_response(result)
+        end
+
+        # Redirect either to referer back or to the desired path
+        handle_block_result Controllers::Responses::Redirect do |result|
+          # Map redirect flashes (if any) to Roda flash messages
+          result.flashes.each { |key, value| flash[key] = value }
+
+          response.redirect result.back? ? request.referer : root_path(result.path)
+        end
+
         # Display appropriate error specific to a given error type
         plugin :error_handler, classes: [
-          ::Karafka::Web::Errors::Ui::NotFoundError,
           ::Rdkafka::RdkafkaError,
+          Errors::Ui::NotFoundError,
           Errors::Ui::ProOnlyError
         ] do |e|
           @error = true
 
           if e.is_a?(Errors::Ui::ProOnlyError)
+            response.status = 402
             view 'shared/exceptions/pro_only'
           else
             response.status = 404
@@ -65,11 +90,36 @@ module Karafka
           view 'shared/exceptions/not_found'
         end
 
+        before do
+          check_csrf!
+        end
+
+        plugin :class_matchers
+
+        # Time matcher with optional hours, minutes and seconds
+        TIME_MATCHER = %r{(\d{4}-\d{2}-\d{2}/?(\d{2})?(:\d{2})?(:\d{2})?)}
+
+        private_constant :TIME_MATCHER
+
+        # Match a date-time. Useful for time-related routes
+        # @note In case the date-time is invalid, raise and render 404
+        # @note The time component is optional as `Time#parse` will fallback to lowest time
+        #   available, so we can build only date based lookups
+        class_matcher(Time, TIME_MATCHER) do |datetime|
+          [Time.parse(datetime)]
+        rescue ArgumentError
+          raise Errors::Ui::NotFoundError
+        end
+
         # Allows us to build current path with additional params
         # @param query_data [Hash] query params we want to add to the current path
         path :current do |query_data = {}|
-          q = query_data.map { |k, v| "#{k}=#{CGI.escape(v.to_s)}" }.join('&')
-          "#{request.path}?#{q}"
+          q = query_data
+              .select { |_, v| v }
+              .map { |k, v| "#{k}=#{CGI.escape(v.to_s)}" }
+              .join('&')
+
+          [request.path, q].compact.delete_if(&:empty?).join('?')
         end
 
         # Sets appropriate template variables based on the response object and renders the

data/lib/karafka/web/ui/controllers/base.rb

@@ -12,6 +12,8 @@ module Karafka
             @params = params
           end
 
+          private
+
           # Builds the respond data object with assigned attributes based on instance variables.
           #
           # @return [Responses::Data] data that should be used to render appropriate view
@@ -33,6 +35,32 @@ module Karafka
               attributes
             )
           end
+
+          # Builds a redirect data object with assigned flashes (if any)
+          # @param path [String, Symbol] relative (without root path) path where we want to be
+          #   redirected or `:back` to use referer back
+          # @param flashes [Hash] hash where key is the flash type and value is the message
+          # @return [Responses::Redirect] redirect result
+          def redirect(path = :back, flashes = {})
+            Responses::Redirect.new(path, flashes)
+          end
+
+          # Initializes the expected pagination engine and assigns expected arguments
+          # @param args Any arguments accepted by the selected pagination engine
+          def paginate(*args)
+            engine = case args.count
+                     when 2
+                       Ui::Lib::Paginations::PageBased
+                     when 3
+                       Ui::Lib::Paginations::WatermarkOffsetsBased
+                     when 4
+                       Ui::Lib::Paginations::OffsetBased
+                     else
+                       raise ::Karafka::Errors::UnsupportedCaseError, args.count
+                     end
+
+            @pagination = engine.new(*args)
+          end
         end
       end
     end

data/lib/karafka/web/ui/controllers/become_pro.rb

@@ -6,7 +6,7 @@ module Karafka
       module Controllers
         # Pro message reporting info controller
         class BecomePro < Base
-          # Display a message, that a give feature is available only in Pro
+          # Display a message, that a given feature is available only in Pro
           def show
             respond
           end