karafka-web 0.5.2 → 0.6.1

data/lib/karafka/web/tracking/contracts/error.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Contracts
+        # Contract for error reporting
+        # Since producers and consumers report their errors to the same topic, we need to have
+        # a unified contract for both
+        class Error < Base
+          configure
+
+          required(:schema_version) { |val| val.is_a?(String) }
+          required(:type) { |val| val.is_a?(String) && !val.empty? }
+          required(:error_class) { |val| val.is_a?(String) && !val.empty? }
+          required(:error_message) { |val| val.is_a?(String) }
+          required(:backtrace) { |val| val.is_a?(String) }
+          required(:details) { |val| val.is_a?(Hash) }
+          required(:occurred_at) { |val| val.is_a?(Float) }
+
+          nested(:process) do
+            required(:name) { |val| val.is_a?(String) && !val.empty? }
+            # Tags may not be present for producers because they may operate from outside of
+            # karafka taggable process
+            optional(:tags) { |val| val.is_a?(Karafka::Core::Taggable::Tags) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/helpers/error_info.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      # Namespace for tracking related helpers
+      module Helpers
+        # Module containing some helper methods useful for extracting extra errors info
+        module ErrorInfo
+          # Extracts the basic error info
+          #
+          # @param error [StandardError] error that occurred
+          # @return [Array<String, String, String>] array with error name, message and backtrace
+          def extract_error_info(error)
+            app_root = "#{::Karafka.root}/"
+
+            gem_home = if ENV.key?('GEM_HOME')
+                         ENV['GEM_HOME']
+                       else
+                         File.expand_path(File.join(Karafka.gem_root.to_s, '../'))
+                       end
+
+            gem_home = "#{gem_home}/"
+
+            backtrace = error.backtrace || []
+            backtrace.map! { |line| line.gsub(app_root, '') }
+            backtrace.map! { |line| line.gsub(gem_home, '') }
+
+            [
+              error.class.name,
+              extract_error_message(error),
+              backtrace.join("\n")
+            ]
+          end
+
+          # @param error [StandardError] error that occurred
+          # @return [String] formatted exception message
+          def extract_error_message(error)
+            error_message = error.message.to_s[0, 10_000]
+            error_message.force_encoding('utf-8')
+            error_message.scrub! if error_message.respond_to?(:scrub!)
+            error_message
+          rescue StandardError
+            '!!! Error message extraction failed !!!'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/memoized_shell.rb

@@ -6,7 +6,7 @@ module Karafka
     # processes
     module Tracking
       # Class used to run shell command that also returns previous result in case of a failure
-      # This is used because children cat get signals when performing stat fetches and then
+      # This is used because children can get signals when performing stat fetches and then
       # fetch is stopped. This can cause invalid results from sub-shell commands.
       #
       # This will return last result as log as there was one.

data/lib/karafka/web/tracking/producers/listeners/base.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Producers
+        # Namespace for producers listeners
+        module Listeners
+          # Base listener for producer related listeners
+          class Base
+            include ::Karafka::Core::Helpers::Time
+            extend Forwardable
+
+            def_delegators :sampler, :track
+            def_delegators :reporter, :report, :report!
+
+            private
+
+            # @return [Object] sampler in use
+            def sampler
+              @sampler ||= ::Karafka::Web.config.tracking.producers.sampler
+            end
+
+            # @return [Object] reporter in use
+            def reporter
+              @reporter ||= ::Karafka::Web.config.tracking.producers.reporter
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/producers/listeners/errors.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Producers
+        module Listeners
+          # Listener for tracking producers published errors
+          class Errors < Base
+            include Tracking::Helpers::ErrorInfo
+
+            # Schema used by producers error reporting
+            SCHEMA_VERSION = '1.0.0'
+
+            private_constant :SCHEMA_VERSION
+
+            # Tracks any producer related errors
+            #
+            # @param event [Karafka::Core::Monitoring::Event]
+            def on_error_occurred(event)
+              track do |sampler|
+                sampler.errors << build_error_details(event)
+              end
+            end
+
+            private
+
+            # @param event [Karafka::Core::Monitoring::Event]
+            # @return [Hash] hash with error data for the sampler
+            def build_error_details(event)
+              type = event[:type]
+
+              error_class, error_message, backtrace = extract_error_info(event[:error])
+
+              {
+                schema_version: SCHEMA_VERSION,
+                producer_id: event[:producer_id],
+                type: type,
+                error_class: error_class,
+                error_message: error_message,
+                backtrace: backtrace,
+                details: build_details(type, event.payload),
+                occurred_at: float_now,
+                process: {
+                  name: sampler.process_name
+                }
+              }
+            end
+
+            # @param type [String] error type
+            # @param payload [Hash] error payload
+            # @return [Hash] hash with details
+            def build_details(type, payload)
+              case type
+              when 'librdkafka.dispatch_error'
+                payload.slice(:topic, :partition, :offset)
+              else
+                {}
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/producers/listeners/reporter.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Producers
+        module Listeners
+          # Special listener that we use to report data about producers states
+          # We don't have to have a separate thread for reporting, because producers have their
+          # own internal threads for changes polling and we can utilize this thread
+          class Reporter < Base
+            # @param _event [Karafka::Core::Monitoring::Event]
+            def on_statistics_emitted(_event)
+              reporter.report
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/producers/reporter.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Producers
+        # Reports the collected data about the producer and sends it, so we can use it in the UI
+        #
+        # @note Producer reported does not have to operate with the `forced` dispatch mainly
+        #   because there is no expectation on immediate status updates for producers and their
+        #   dispatch flow is always periodic based.
+        class Reporter
+          include ::Karafka::Core::Helpers::Time
+
+          # Minimum number of messages to produce to produce them in sync mode
+          # This acts as a small back-off not to overload the system in case we would have
+          # extremely big number of errors happening
+          PRODUCE_SYNC_THRESHOLD = 25
+
+          private_constant :PRODUCE_SYNC_THRESHOLD
+
+          # This mutex is shared between tracker and samplers so there is no case where metrics
+          # would be collected same time tracker reports
+          MUTEX = Mutex.new
+
+          def initialize
+            # If there are any errors right after we started sampling, dispatch them immediately
+            @tracked_at = monotonic_now - 10_000
+            @error_contract = Tracking::Contracts::Error.new
+          end
+
+          # Dispatches the current state from sampler to appropriate topics
+          def report
+            MUTEX.synchronize do
+              return unless report?
+
+              @tracked_at = monotonic_now
+
+              # Report errors that occurred (if any)
+              messages = sampler.errors.map do |error|
+                @error_contract.validate!(error)
+
+                {
+                  topic: Karafka::Web.config.topics.errors,
+                  payload: error.to_json,
+                  # Always dispatch errors from the same process to the same partition
+                  key: error[:process][:name]
+                }
+              end
+
+              return if messages.empty?
+
+              produce(messages)
+
+              # Clear the sampler so it tracks new state changes without previous once impacting
+              # the data
+              sampler.clear
+            end
+          end
+
+          private
+
+          # @return [Boolean] Should we report or is it not yet time to do so
+          def report?
+            return false unless ::Karafka.producer.status.active?
+
+            (monotonic_now - @tracked_at) >= ::Karafka::Web.config.tracking.interval
+          end
+
+          # @return [Object] sampler for the metrics
+          def sampler
+            @sampler ||= ::Karafka::Web.config.tracking.producers.sampler
+          end
+
+          # Produces messages to Kafka.
+          #
+          # @param messages [Array<Hash>]
+          #
+          # @note We pick either sync or async dependent on number of messages. The trick here is,
+          #   that we do not want to end up overloading the internal queue with messages in case
+          #   someone has a lot of errors from processing or other errors. Producing sync will wait
+          #   for the delivery, hence will slow things down a little bit. On the other hand during
+          #   normal operations we should not have that many messages to dispatch and it should not
+          #   slowdown any processing.
+          def produce(messages)
+            if messages.count >= PRODUCE_SYNC_THRESHOLD
+              ::Karafka.producer.produce_many_sync(messages)
+            else
+              ::Karafka.producer.produce_many_async(messages)
+            end
+          # Since we run this in a background thread, there may be a case upon shutdown, where the
+          # producer is closed right before a potential dispatch. It is not worth dealing with this
+          # and we can just safely ignore this
+          rescue WaterDrop::Errors::ProducerClosedError
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/producers/sampler.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      # Namespace for all the things related to tracking producers
+      module Producers
+        # Samples for collecting producers related data we're interested in
+        class Sampler < Tracking::Sampler
+          include ::Karafka::Core::Helpers::Time
+
+          attr_reader :errors
+
+          # Current schema version
+          # This can be used in the future for detecting incompatible changes and writing
+          # migrations
+          SCHEMA_VERSION = '1.0.0'
+
+          def initialize
+            super
+
+            @errors = []
+            @started_at = float_now
+          end
+
+          # We cannot report and track the same time, that is why we use mutex here. To make sure
+          # that samples aggregations and counting does not interact with reporter flushing.
+          def track
+            Reporter::MUTEX.synchronize do
+              yield(self)
+            end
+          end
+
+          # Clears the sampler (for use after data dispatch)
+          def clear
+            @errors.clear
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/sampler.rb

@@ -32,6 +32,11 @@ module Karafka
           ::Karafka::VERSION
         end
 
+        # @return [String] Karafka Web UI version
+        def karafka_web_version
+          ::Karafka::Web::VERSION
+        end
+
         # @return [String] Karafka::Core version
         def karafka_core_version
           ::Karafka::Core::VERSION

data/lib/karafka/web/ui/controllers/consumers.rb

@@ -10,11 +10,9 @@ module Karafka
           # @note For now we load all and paginate over the squashed data.
           def index
             @current_state = Models::State.current!
-            processes_total = Models::Processes.active(@current_state)
-
-            @counters = Lib::HashProxy.new(@current_state[:stats])
+            @counters = Models::Counters.new(@current_state)
             @processes, @next_page = Lib::PaginateArray.new.call(
-              processes_total,
+              Models::Processes.active(@current_state),
               @params.current_page
             )
 

data/lib/karafka/web/ui/models/counters.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Models
+        # Represents the top counters bar values on the consumers view
+        class Counters < Lib::HashProxy
+          # Max errors partitions we support for estimations
+          MAX_ERROR_PARTITIONS = 100
+
+          private_constant :MAX_ERROR_PARTITIONS
+
+          # @param state [Hash]
+          def initialize(state)
+            super(state[:stats])
+            @hash[:errors] = estimate_errors_count
+          end
+
+          private
+
+          # Estimates the number of errors present in the errors topic.
+          def estimate_errors_count
+            estimated = 0
+
+            MAX_ERROR_PARTITIONS.times do |partition|
+              begin
+                offsets = Karafka::Admin.read_watermark_offsets(
+                  ::Karafka::Web.config.topics.errors,
+                  partition
+                )
+              # We estimate that way instead of using `#cluster_info` to get the partitions count
+              # inside the errors topic, because it is around 90x faster to query for invalid
+              # partition and get the error, instead of querying for all topics on a big cluster
+              #
+              # Most of the users use one or few error partitions at most, so this is fairly
+              # efficient and not problematic
+              rescue Rdkafka::RdkafkaError => e
+                e.code == :unknown_partition ? break : raise
+              end
+
+              estimated += offsets.last - offsets.first
+            end
+
+            estimated
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/models/status.rb

@@ -5,6 +5,7 @@ module Karafka
     module Ui
       module Models
         # Model that represents the general status of the Web UI.
+        #
         # We use this data to display a status page that helps with debugging on what is missing
         # in the overall setup of the Web UI.
         #
@@ -15,7 +16,18 @@ module Karafka
           Step = Struct.new(:status, :details) do
             # @return [Boolean] is the given step successfully configured and working
             def success?
-              status == :success
+              status == :success || status == :warning
+            end
+
+            # @return [String] local namespace for partial of a given type
+            def partial_namespace
+              case status
+              when :success then 'successes'
+              when :warning then 'warnings'
+              when :failure then 'failures'
+              else
+                raise ::Karafka::Errors::UnsupportedCaseError, status
+              end
             end
 
             # @return [String] stringified status
@@ -29,11 +41,21 @@ module Karafka
             connect
           end
 
-          # @return [Status::Step] were we able to connect to Kafka or not
+          # @return [Status::Step] were we able to connect to Kafka or not and how fast.
+          # Some people try to work with Kafka over the internet with really high latency and this
+          # should be highlighted in the UI as often the connection just becomes unstable
           def connection
+            level = if @connection_time < 1_000
+                      :success
+                    elsif @connection_time < 1_000_000
+                      :warning
+                    else
+                      :failure
+                    end
+
             Step.new(
-              @connected ? :success : :failure,
-              nil
+              level,
+              { time: @connection_time }
             )
           end
 
@@ -171,12 +193,14 @@ module Karafka
             topics
           end
 
-          # Tries connecting with the cluster and sets the connection state
+          # Tries connecting with the cluster and saves the cluster info and the connection time
+          # @note If fails, `connection_time` will be 1_000_000
           def connect
+            started = Time.now.to_f
             @cluster_info = ::Karafka::Admin.cluster_info
-            @connected = true
+            @connection_time = (Time.now.to_f - started) * 1_000
           rescue ::Rdkafka::RdkafkaError
-            @connected = false
+            @connection_time = 1_000_000
           end
         end
       end

data/lib/karafka/web/ui/pro/controllers/consumers.rb

@@ -21,10 +21,9 @@ module Karafka
             # Consumers list
             def index
               @current_state = Models::State.current!
-              processes_total = Models::Processes.active(@current_state)
-              @counters = Lib::HashProxy.new(@current_state[:stats])
+              @counters = Models::Counters.new(@current_state)
               @processes, @next_page = Lib::PaginateArray.new.call(
-                processes_total,
+                Models::Processes.active(@current_state),
                 @params.current_page
               )
 

data/lib/karafka/web/ui/pro/views/consumers/consumer/_job.erb

@@ -1,4 +1,10 @@
 <tr>
+  <td>
+    <span class="badge bg-secondary badge-topic" title="Consumer group: <%= job.consumer_group %>">
+      <%= job.topic %>:
+      <%= job.partition %>
+    </span>
+  </td>
   <td>
     <code><%= job.consumer %></code>
 
@@ -7,12 +13,6 @@
       <%== tags(job.tags) %>
     <% end %>
   </td>
-  <td>
-    <span class="badge bg-secondary badge-topic" title="Consumer group: <%= job.consumer_group %>">
-      <%= job.topic %>:
-      <%= job.partition %>
-    </span>
-  </td>
   <td>
     <code>#<%= job.type %></code>
   </td>

data/lib/karafka/web/ui/pro/views/consumers/consumer/_metrics.erb

@@ -113,7 +113,12 @@
               </span>
 
               <span class="badge bg-secondary">
-                karafka core
+                karafka-web
+                <%= @process.karafka_web %>
+              </span>
+
+              <span class="badge bg-secondary">
+                karafka-core
                 <%= @process.karafka_core %>
               </span>
 

data/lib/karafka/web/ui/pro/views/consumers/index.erb

@@ -4,27 +4,31 @@
 <div class="container">
   <div class="row">
     <div class="col-sm-12">
-      <table class="processes bg-white table table-hover table-bordered table-striped mb-0 align-middle">
-        <thead>
-          <tr class="align-middle">
-            <th>Name</th>
-            <th class="col-sm-2">Started</th>
-            <th class="col-sm-1">Memory</th>
-            <th class="col-sm-1">Performance</th>
-            <th class="col-sm-1">Load</th>
-            <th class="col-sm-1">Total lag</th>
-          </tr>
-        </thead>
-        <tbody>
-          <%==
-            render_each(
-              @processes,
-              'consumers/_consumer',
-              local: :process
-            )
-          %>
-        </tbody>
-      </table>
+      <% if @processes.empty? %>
+        <%== partial 'consumers/no_consumers' %>
+      <% else %>
+        <table class="processes bg-white table table-hover table-bordered table-striped mb-0 align-middle">
+          <thead>
+            <tr class="align-middle">
+              <th>Name</th>
+              <th class="col-sm-2">Started</th>
+              <th class="col-sm-1">Memory</th>
+              <th class="col-sm-1">Performance</th>
+              <th class="col-sm-1">Load</th>
+              <th class="col-sm-1">Total lag</th>
+            </tr>
+          </thead>
+          <tbody>
+            <%==
+              render_each(
+                @processes,
+                'consumers/_consumer',
+                local: :process
+              )
+            %>
+          </tbody>
+        </table>
+      <% end %>
     </div>
   </div>
 </div>

data/lib/karafka/web/ui/pro/views/consumers/jobs.erb

@@ -17,8 +17,8 @@
         <table class="processes bg-white table table-hover table-bordered table-striped mb-0 align-middle">
           <thead>
             <tr class="align-middle">
-              <th>Consumer</th>
               <th>Topic</th>
+              <th>Consumer</th>
               <th>Type</th>
               <th>First offset</th>
               <th>Last offset</th>

data/lib/karafka/web/ui/pro/views/errors/_breadcrumbs.erb

@@ -16,9 +16,8 @@
       <%=
         type = @error_message.payload[:type]
         error_class = @error_message.payload[:error_class]
-        offset = @error_message.offset
 
-        "#{type}: #{error_class} #{offset}"
+        "#{type}: #{error_class}"
       %>
     </a>
   </li>

data/lib/karafka/web/ui/pro/views/errors/_error.erb

@@ -9,14 +9,16 @@
 
   <tr>
     <td>
-      <% if error[:details].key?(:topic) %>
-        <%= error[:details][:topic] %>: <%= error[:details][:partition] %>
-      <% else %>
-        <%= error[:type] %>
-      <% end %>
+      <span class="badge bg-secondary badge-topic">
+        <% if error[:details].key?(:topic) %>
+          <%= error[:details][:topic] %>: <%= error[:details][:partition] %>
+        <% else %>
+          <%= error[:type] %>
+        <% end %>
+      </span>
     </td>
     <td>
-      <%== error[:process_name] %>
+      <%== error[:process][:name] %>
     </td>
     <td>
       <%= error[:error_class] %>: