karafka-web 0.1.0

data/lib/karafka/web/tracking/consumers/sampler.rb

@@ -0,0 +1,216 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      # Namespace for all the things related to tracking consumers and consuming processes
+      module Consumers
+        # Samples for fetching and storing metrics samples about the consumer process
+        class Sampler
+          include ::Karafka::Core::Helpers::Time
+
+          attr_reader :counters, :consumer_groups, :errors, :times, :pauses, :jobs
+
+          # 60 seconds window for time tracked window-based metrics
+          TIMES_TTL = 60
+
+          # Times ttl in ms
+          TIMES_TTL_MS = TIMES_TTL * 1_000
+
+          private_constant :TIMES_TTL, :TIMES_TTL_MS
+
+          def initialize
+            @counters = {
+              batches: 0,
+              messages: 0,
+              errors: 0,
+              retries: 0,
+              dead: 0
+            }
+            @times = TtlHash.new(TIMES_TTL_MS)
+            @consumer_groups = {}
+            @errors = []
+            @started_at = float_now
+            @pauses = Set.new
+            @jobs = {}
+            @shell = MemoizedShell.new
+          end
+
+          # We cannot report and track the same time, that is why we use mutex here. To make sure
+          # that samples aggregations and counting does not interact with reporter flushing.
+          def track
+            Reporter::MUTEX.synchronize do
+              yield(self)
+            end
+          end
+
+          # @return [Hash] report hash with all the details about consumer operations
+          def to_report
+            {
+              schema_version: '1.0.0',
+              type: 'consumer',
+              dispatched_at: float_now,
+
+              process: {
+                started_at: @started_at,
+                name: process_name,
+                status: ::Karafka::App.config.internal.status.to_s,
+                listeners: Karafka::Server.listeners.count,
+                concurrency: concurrency,
+                memory_usage: memory_usage,
+                memory_total_usage: memory_total_usage,
+                memory_size: memory_size,
+                cpu_count: cpu_count,
+                cpu_usage: cpu_usage
+              },
+
+              versions: {
+                ruby: ruby_version,
+                karafka: ::Karafka::VERSION,
+                waterdrop: ::WaterDrop::VERSION
+              },
+
+              stats: Karafka::Server.jobs_queue.statistics.merge(
+                utilization: utilization
+              ).merge(total: @counters),
+
+              consumer_groups: @consumer_groups,
+              jobs: jobs.values
+            }
+          end
+
+          # Clears counters and errors.
+          # Used after data is reported by reported to start collecting new samples
+          # @note We do not clear processing or pauses or other things like this because we track
+          #   their states and not values, so they need to be tracked between flushes.
+          def clear
+            @counters.each { |k, _| @counters[k] = 0 }
+
+            @errors.clear
+          end
+
+          private
+
+          # @return [Numeric] % utilization of all the threads. 100% means all the threads are
+          #   utilized all the time within the given time window. 0% means, nothing is happening
+          #   most if not all the time.
+          def utilization
+            return 0 if times[:total].empty?
+
+            # Max times ttl
+            timefactor = float_now - @started_at
+            timefactor = timefactor > TIMES_TTL ? TIMES_TTL : timefactor
+
+            # We divide by 1_000 to convert from milliseconds
+            # We multiply by 100 to have it in % scale
+            times[:total].sum / 1_000 / concurrency / timefactor * 100
+          end
+
+          # @return [String] Unique process name
+          def process_name
+            @process_name ||= "#{Socket.gethostname}:#{::Process.pid}:#{SecureRandom.hex(6)}"
+          end
+
+          # @return [Integer] memory used by this process in kilobytes
+          def memory_usage
+            pid = ::Process.pid
+
+            case RUBY_PLATFORM
+            # Reading this that way is cheaper than running a shell command
+            when /linux/
+              IO.readlines("/proc/#{pid}/status").each do |line|
+                next unless line.start_with?('VmRSS:')
+
+                break line.split[1].to_i
+              end
+            when /darwin|bsd/
+              @shell
+                .call("ps -o pid,rss -p #{pid}")
+                .lines
+                .last
+                .split
+                .last
+                .to_i
+            else
+              0
+            end
+          end
+
+          # Total memory used in the OS
+          def memory_total_usage
+            case RUBY_PLATFORM
+            when /darwin|bsd|linux/
+              @shell
+                .call('ps -A -o rss=')
+                .split("\n")
+                .inject { |a, e| a.to_i + e.strip.to_i }
+            else
+              0
+            end
+          end
+
+          # @return [Integer] total amount of memory
+          def memory_size
+            @memory_size ||= case RUBY_PLATFORM
+                             when /linux/
+                               @shell
+                                 .call('grep MemTotal /proc/meminfo')
+                                 .match(/(\d+)/)
+                                 .to_s
+                                 .to_i
+                             when /darwin|bsd/
+                               @shell
+                                 .call('sysctl -a')
+                                 .split("\n")
+                                 .find { |line| line.start_with?('hw.memsize:') }
+                                 .to_s
+                                 .split(' ')
+                                 .last
+                                 .to_i
+                             else
+                               0
+                             end
+          end
+
+          # @return [Array<Float>] load averages for last 1, 5 and 15 minutes
+          def cpu_usage
+            case RUBY_PLATFORM
+            when /darwin|bsd|linux/
+              @shell
+                .call('w | head -1')
+                .strip
+                .split(' ')
+                .map(&:to_f)
+                .last(3)
+            else
+              [-1, -1, -1]
+            end
+          end
+
+          # @return [Integer] CPU count
+          def cpu_count
+            @cpu_count ||= Etc.nprocessors
+          end
+
+          # @return [String] currently used ruby version with details
+          def ruby_version
+            if defined?(JRUBY_VERSION)
+              revision = JRUBY_REVISION.to_s
+              version = JRUBY_VERSION
+            else
+              revision = RUBY_REVISION.to_s
+              version = RUBY_ENGINE_VERSION
+            end
+
+            "#{RUBY_ENGINE} #{version}-#{RUBY_PATCHLEVEL} #{revision[0..5]}"
+          end
+
+          # @return [Integer] number of threads that process work
+          def concurrency
+            @concurrency ||= Karafka::App.config.concurrency
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/memoized_shell.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    # Namespace used to encapsulate all components needed to track and report states of particular
+    # processes
+    module Tracking
+      # Class used to run shell command that also returns previous result in case of a failure
+      # This is used because children cat get signals when performing stat fetches and then
+      # fetch is stopped. This can cause invalid results from sub-shell commands.
+      #
+      # This will return last result as log as there was one.
+      class MemoizedShell
+        # Hpw many tries do we want to perform before giving up on the shell command
+        MAX_ATTEMPTS = 4
+
+        private_constant :MAX_ATTEMPTS
+
+        def initialize
+          @accu = {}
+        end
+
+        # @param cmd [String]
+        # @return [String, nil] sub-shell evaluation string result or nil if we were not able to
+        #   run or re-run the call.
+        def call(cmd)
+          attempt ||= 0
+
+          while attempt < MAX_ATTEMPTS
+            attempt += 1
+
+            stdout_str, status = Open3.capture2(cmd)
+
+            if status.success?
+              @accu[cmd] = stdout_str
+              return stdout_str
+            else
+              return stdout_str if attempt > MAX_ATTEMPTS
+              return @accu[cmd] if @accu.key?(cmd)
+            end
+          end
+
+          @accu[cmd]
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/reporter.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      # Reports the collected data about the process and sends it, so we can use it in the UI
+      class Reporter
+        include ::Karafka::Core::Helpers::Time
+        include ::Karafka::Helpers::Async
+
+        # Minimum number of messages to produce to produce then in sync mode
+        # This acts as a small back-off not to overload the system in case we would have extremely
+        # big number of errors happening
+        PRODUCE_SYNC_THRESHOLD = 25
+
+        private_constant :PRODUCE_SYNC_THRESHOLD
+
+        # This mutex is shared between tracker and samplers so there is no case where metrics
+        # would be collected same time tracker reports
+        MUTEX = Mutex.new
+
+        def initialize
+          # Move back so first report is dispatched fast to indicate, that the process is alive
+          @tracked_at = monotonic_now - 10_000
+          @consumer_contract = Consumers::Contracts::Report.new
+        end
+
+        # Dispatches the current state from sampler to appropriate topics
+        #
+        # @param forced [Boolean] should we report bypassing the time frequency or should we report
+        #   only in case we would not send the report for long enough time.
+        def report(forced: false)
+          MUTEX.synchronize do
+            # Start background thread only when needed
+            # This prevents us from starting it too early or for non-consumer processes where
+            # Karafka is being included
+            async_call unless @running
+
+            return unless report?(forced)
+
+            @tracked_at = monotonic_now
+
+            consumer_report = consumer_sampler.to_report
+
+            @consumer_contract.validate!(consumer_report)
+
+            # Report consumers statuses
+            messages = [
+              {
+                topic: ::Karafka::Web.config.topics.consumers.reports,
+                payload: consumer_report.to_json,
+                key: consumer_report[:process][:name]
+              }
+            ]
+
+            # Report errors that occurred (if any)
+            messages += consumer_sampler.errors.map do |error|
+              {
+                topic: Karafka::Web.config.topics.errors,
+                # Inject process name into the error details for easier tracing
+                payload: error.merge(
+                  process_name: consumer_report[:process][:name]
+                ).to_json,
+                # Always dispatch errors from the same process to the same partition
+                key: consumer_report[:process][:name]
+              }
+            end
+
+            produce(messages)
+
+            # Clear the sampler so it tracks new state changes without previous once impacting
+            # the data
+            consumer_sampler.clear
+          end
+        # Since we run this in a background thread, there may be a case upon shutdown, where the
+        # producer is closed right before a potential dispatch. It is not worth dealing with this
+        # and we can just safely ignore this
+        rescue WaterDrop::Errors::ProducerClosedError
+          nil
+        end
+
+        # Reports bypassing frequency check. This can be used to report when state changes in the
+        # process drastically. For example when process is stopping, we want to indicate this as
+        # fast as possible in the UI, etc.
+        def report!
+          report(forced: true)
+        end
+
+        private
+
+        # Reports the process state once in a while
+        def call
+          @running = true
+
+          loop do
+            report
+
+            # We won't track more often anyhow but want to try frequently not to miss a window
+            # We need to convert the sleep interval into seconds for sleep
+            sleep(::Karafka::Web.config.tracking.interval / 1_000 / 10)
+          end
+        end
+
+        # @param forced [Boolean] is this report forced. Forced means that as long as we can flush
+        #   we will flush
+        # @return [Boolean] Should we report or is it not yet time to do so
+        def report?(forced)
+          # We never report in initializing phase because things are not yet fully configured
+          return false if ::Karafka::App.initializing?
+          # We never report in the initialized because server is not yet ready until Karafka is
+          # fully running and some of the things like listeners are not yet available
+          return false if ::Karafka::App.initialized?
+
+          return true if forced
+
+          (monotonic_now - @tracked_at) >= ::Karafka::Web.config.tracking.interval
+        end
+
+        # @return [Object] sampler for the metrics
+        def consumer_sampler
+          @consumer_sampler ||= ::Karafka::Web.config.tracking.consumers.sampler
+        end
+
+        # Produces messages to Kafka.
+        #
+        # @param messages [Array<Hash>]
+        #
+        # @note We pick either sync or async dependent on number of messages. The trick here is,
+        #   that we do not want to end up overloading the internal queue with messages in case
+        #   someone has a lot of errors from processing or other errors. Producing sync will wait
+        #   for the delivery, hence will slow things down a little bit. On the other hand during
+        #   normal operations we should not have that many messages to dispatch and it should not
+        #   slowdown any processing.
+        def produce(messages)
+          if messages.count >= PRODUCE_SYNC_THRESHOLD
+            ::Karafka.producer.produce_many_sync(messages)
+          else
+            ::Karafka.producer.produce_many_async(messages)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/ttl_array.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      # Array that allows us to store data points that expire over time automatically.
+      class TtlArray
+        include ::Karafka::Core::Helpers::Time
+        include Enumerable
+
+        # @param ttl [Integer] milliseconds ttl
+        def initialize(ttl)
+          @ttl = ttl
+          @accu = []
+        end
+
+        # Iterates over only active elements
+        def each
+          clear
+
+          @accu.each do |sample|
+            yield sample[:value]
+          end
+        end
+
+        # @param value [Object] adds value to the array
+        # @return [Object] added element
+        def <<(value)
+          @accu << { value: value, added_at: monotonic_now }
+
+          clear
+
+          value
+        end
+
+        # @return [Boolean] is the array empty
+        def empty?
+          clear
+          @accu.empty?
+        end
+
+        # @return [Array] pure array version with only active elements
+        def to_a
+          clear
+          super
+        end
+
+        private
+
+        # Evicts outdated samples
+        def clear
+          @accu.delete_if do |sample|
+            monotonic_now - sample[:added_at] > @ttl
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/ttl_hash.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      # Hash that accumulates data that has an expiration date (ttl)
+      # Used to keep track of metrics in a window
+      class TtlHash < Hash
+        # @param ttl [Integer] milliseconds ttl
+        def initialize(ttl)
+          super() { |k, v| k[v] = TtlArray.new(ttl) }
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/app.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    # Web UI namespace
+    module Ui
+      # Main Roda Web App that servers all the metrics and stats
+      class App < Base
+        # Use the gem views and assets location
+        opts[:root] = Karafka::Web.gem_root.join('lib/karafka/web/ui')
+
+        instance_exec(&CONTEXT_DETAILS)
+
+        route do |r|
+          r.root { r.redirect root_path('consumers') }
+
+          @current_page = params.current_page
+
+          r.on 'consumers' do
+            r.get String, 'subscriptions' do |_process_id|
+              raise Errors::Ui::ProOnlyError
+            end
+
+            r.get do
+              @breadcrumbs = false
+              controller = Controllers::Consumers.new(params)
+              render_response controller.index
+            end
+          end
+
+          %w[
+            health
+            explorer
+            dlq
+          ].each do |route|
+            r.get route, [String, true], [String, true] do
+              raise Errors::Ui::ProOnlyError
+            end
+          end
+
+          r.get 'jobs' do
+            controller = Controllers::Jobs.new(params)
+            render_response controller.index
+          end
+
+          r.on 'routing' do
+            controller = Controllers::Routing.new(params)
+
+            r.get String do |topic_id|
+              render_response controller.show(topic_id)
+            end
+
+            r.get do
+              render_response controller.index
+            end
+          end
+
+          r.get 'cluster' do
+            controller = Controllers::Cluster.new(params)
+            render_response controller.index
+          end
+
+          r.on 'errors' do
+            controller = Controllers::Errors.new(params)
+
+            r.get Integer do |offset|
+              render_response controller.show(offset)
+            end
+
+            r.get do
+              render_response controller.index
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/base.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      # Base Roda application
+      class Base < Roda
+        include Helpers::ApplicationHelper
+
+        # Details that need to be evaluated in the context of OSS or Pro web UI.
+        # If those would be evaluated in the base, they would not be initialized as expected
+        CONTEXT_DETAILS = lambda do
+          plugin(
+            :static,
+            %w[/javascripts /stylesheets /images],
+            root: Karafka::Web.gem_root.join('lib/karafka/web/ui/public')
+          )
+          plugin :render_each
+          plugin :partials
+        end
+
+        plugin :render, escape: true, engine: 'erb'
+        plugin :run_append_slash
+        plugin :error_handler
+        plugin :not_found
+        plugin :path
+
+        # Display appropriate error specific to a given error type
+        plugin :error_handler, classes: [
+          ::Karafka::Web::Errors::Ui::NotFoundError,
+          ::Rdkafka::RdkafkaError,
+          Errors::Ui::ProOnlyError
+        ] do |e|
+          @error = true
+
+          if e.is_a?(Errors::Ui::ProOnlyError)
+            view 'shared/exceptions/pro_only'
+          else
+            response.status = 404
+            view 'shared/exceptions/not_found'
+          end
+        end
+
+        not_found do
+          @error = true
+          response.status = 404
+          view 'shared/exceptions/not_found'
+        end
+
+        # Allows us to build current path with additional params
+        # @param query_data [Hash] query params we want to add to the current path
+        path :current do |query_data = {}|
+          q = query_data.map { |k, v| "#{k}=#{CGI.escape(v.to_s)}" }.join('&')
+          "#{request.path}?#{q}"
+        end
+
+        # Sets appropriate template variables based on the response object and renders the
+        # expected view
+        # @param response [Karafka::Web::Ui::Controllers::Responses::Data] response data object
+        def render_response(response)
+          response.attributes.each do |key, value|
+            instance_variable_set(
+              "@#{key}", value
+            )
+          end
+
+          view(response.path)
+        end
+
+        # @return [Karafka::Web::Ui::Controllers::Requests::Params] curated params
+        def params
+          Controllers::Requests::Params.new(request.params)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/controllers/base.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      # Namespace for controller related components in the Web UI app.
+      module Controllers
+        # Base controller from which all the controllers should inherit.
+        class Base
+          # @param params [Karafka::Web::Ui::Controllers::Requests::Params] request parameters
+          def initialize(params)
+            @params = params
+          end
+
+          # Builds the respond data object with assigned attributes based on instance variables.
+          #
+          # @return [Responses::Data] data that should be used to render appropriate view
+          def respond
+            attributes = {}
+
+            scope = self.class.to_s.split('::').last.gsub(/(.)([A-Z])/, '\1_\2').downcase
+            action = caller_locations(1, 1)[0].label
+
+            instance_variables.each do |iv|
+              next if iv.to_s.start_with?('@_')
+              next if iv.to_s.start_with?('@params')
+
+              attributes[iv.to_s.delete('@').to_sym] = instance_variable_get(iv)
+            end
+
+            Responses::Data.new(
+              "#{scope}/#{action}",
+              attributes
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/controllers/become_pro.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Controllers
+        # Pro message reporting info controller
+        class BecomePro < Base
+          # Display a message, that a give feature is available only in Pro
+          def show
+            respond
+          end
+        end
+      end
+    end
+  end
+end