karafka-web 0.1.0

data/lib/karafka/web/installer.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    # Responsible for setup of the Web UI and Karafka Web-UI related components initialization.
+    class Installer
+      # Creates needed topics and the initial zero state, so even if no `karafka server` processes
+      # are running, we can still display the empty UI
+      #
+      # @param replication_factor [Integer] replication factor we want to use (1 by default)
+      def bootstrap!(replication_factor: 1)
+        bootstrap_topics!(replication_factor)
+        bootstrap_state!
+      end
+
+      # Adds the extra needed consumer group, topics and routes for Web UI to be able to operate
+      def enable!
+        ::Karafka::App.routes.draw do
+          web_deserializer = ::Karafka::Web::Deserializer.new
+
+          consumer_group ::Karafka::Web.config.processing.consumer_group do
+            # Topic we listen on to materialize the states
+            topic ::Karafka::Web.config.topics.consumers.reports do
+              # Since we materialize state in intervals, we can poll for half of this time without
+              # impacting the reporting responsiveness
+              max_wait_time ::Karafka::Web.config.processing.interval / 2
+              max_messages 1_000
+              consumer ::Karafka::Web::Processing::Consumer
+              deserializer web_deserializer
+              manual_offset_management true
+            end
+
+            # We define those two here without consumption, so Web understands how to deserialize
+            # them when used / viewed
+            topic ::Karafka::Web.config.topics.consumers.states do
+              active false
+              deserializer web_deserializer
+            end
+
+            topic ::Karafka::Web.config.topics.errors do
+              active false
+              deserializer web_deserializer
+            end
+          end
+        end
+
+        # Installs all the consumer related listeners
+        ::Karafka::Web.config.tracking.consumers.listeners.each do |listener|
+          ::Karafka.monitor.subscribe(listener)
+        end
+
+        # Installs all the producer related listeners
+        ::Karafka::Web.config.tracking.producers.listeners.each do |listener|
+          ::Karafka.producer.monitor.subscribe(listener)
+        end
+      end
+
+      private
+
+      # Creates all the needed topics for the admin UI to work
+      #
+      # @param replication_factor [Integer]
+      def bootstrap_topics!(replication_factor = 1)
+        # This topic needs to have one partition
+        ::Karafka::Admin.create_topic(
+          ::Karafka::Web.config.topics.consumers.states,
+          1,
+          replication_factor,
+          # We care only about the most recent state, previous are irrelevant
+          { 'cleanup.policy': 'compact' }
+        )
+
+        # This topic needs to have one partition
+        ::Karafka::Admin.create_topic(
+          ::Karafka::Web.config.topics.consumers.reports,
+          1,
+          replication_factor,
+          # We do not need to to store this data for longer than 7 days as this data is only used
+          # to materialize the end states
+          # On the other hand we do not want to have it really short-living because in case of a
+          # consumer crash, we may want to use this info to catch up and backfill the state
+          { 'retention.ms': 7 * 24 * 60 * 60 * 1_000 }
+        )
+
+        # All the errors will be dispatched here
+        # This topic can have multiple partitions but we go with one by default. A single Ruby
+        # process should not crash that often and if there is an expectation of a higher volume
+        # of errors, this can be changed by the end user
+        ::Karafka::Admin.create_topic(
+          ::Karafka::Web.config.topics.errors,
+          1,
+          replication_factor
+        )
+
+        bootstrap_state!
+      end
+
+      # Creates the initial state record with all values being empty
+      def bootstrap_state!
+        ::Karafka.producer.produce_sync(
+          topic: Karafka::Web.config.topics.consumers.states,
+          key: Karafka::Web.config.topics.consumers.states,
+          payload: {
+            processes: {},
+            stats: {
+              batches: 0,
+              messages: 0,
+              errors: 0,
+              retries: 0,
+              dead: 0,
+              busy: 0,
+              enqueued: 0,
+              threads_count: 0,
+              processes: 0,
+              rss: 0,
+              listeners_count: 0,
+              utilization: 0
+            }
+          }.to_json
+        )
+      end
+    end
+  end
+end

data/lib/karafka/web/processing/consumer.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    # Namespace used to encapsulate all the components needed to process the states data and
+    # store it back in Kafka
+    module Processing
+      # Consumer used to squash and process statistics coming from particular processes, so this
+      # data can be read and used. We consume this info overwriting the data we previously had
+      # (if any)
+      class Consumer < Karafka::BaseConsumer
+        include ::Karafka::Core::Helpers::Time
+
+        # @param args [Object] all the arguments `Karafka::BaseConsumer` accepts by default
+        def initialize(*args)
+          super
+
+          @flush_interval = ::Karafka::Web.config.processing.interval / 1_000
+          @consumers_aggregator = ::Karafka::Web.config.processing.consumers.aggregator
+          # We set this that way so we report with first batch and so we report in the development
+          # mode. In the development mode, there is a new instance per each invocation, thus we need
+          # to always initially report, so the web UI works well in the dev mode where consumer
+          # instances are not long-living.
+          @flushed_at = monotonic_now - @flush_interval
+        end
+
+        # Aggregates consumers state into a single current state representation
+        def consume
+          messages
+            .select { |message| message.payload[:type] == 'consumer' }
+            .each { |message| @consumers_aggregator.add(message.payload, message.offset) }
+
+          return unless periodic_flush?
+
+          flush
+
+          mark_as_consumed(messages.last)
+        end
+
+        # Flush final state on shutdown
+        def shutdown
+          flush if @consumers_aggregator
+        end
+
+        private
+
+        # @return [Boolean] is it time to persist the new current state
+        def periodic_flush?
+          (monotonic_now - @flushed_at) > @flush_interval
+        end
+
+        # Persists the new current state by flushing it to Kafka
+        def flush
+          @flushed_at = monotonic_now
+
+          producer.produce_async(
+            topic: Karafka::Web.config.topics.consumers.states,
+            payload: @consumers_aggregator.to_json,
+            # This will ensure that the consumer states are compacted
+            key: Karafka::Web.config.topics.consumers.states
+          )
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/processing/consumers/aggregator.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Processing
+      # Namespace for consumer sub-components
+      module Consumers
+        # Aggregator that tracks consumers processes states, aggregates the metrics and converts
+        # data points into a materialized current state.
+        class Aggregator
+          include ::Karafka::Core::Helpers::Time
+
+          def initialize
+            # We keep whole reports for computation of active, current counters
+            @active_reports = {}
+          end
+
+          # Uses provided process state report to update the current materialized state
+          # @param report [Hash] consumer process state report
+          # @param offset [Integer] offset of the message with the state report. This offset is
+          #   needed as we need to be able to get all the consumers reports from a given offset.
+          def add(report, offset)
+            memoize_process_report(report)
+            increment_total_counters(report)
+            update_process_state(report, offset)
+            # We always evict after counters updates because we want to use expired (stopped)
+            # data for counters as it was valid previously. This can happen only when web consumer
+            # had a lag and is catching up.
+            evict_expired_processes
+            # We could calculate this on a per request basis but this would require fetching all
+            # the active processes for each view and we do not want that for performance reasons
+            refresh_current_stats
+          end
+
+          # @param _args [Object] extra parsing arguments (not used)
+          # @return [String] json representation of the current processes state
+          def to_json(*_args)
+            state.to_json
+          end
+
+          private
+
+          # @return [Hash] hash with current state from Kafka or an empty new initial state
+          def state
+            @state ||= State.current
+          end
+
+          # Updates the report for given process in memory
+          # @param report [Hash]
+          def memoize_process_report(report)
+            @active_reports[report[:process][:name]] = report
+          end
+
+          # Increments the total counters based on the provided report
+          # @param report [Hash]
+          def increment_total_counters(report)
+            report[:stats][:total].each do |key, value|
+              state[:stats][key] ||= 0
+              state[:stats][key] += value
+            end
+          end
+
+          # Registers or updates the given process state based on the report
+          #
+          # @param report [Hash]
+          # @param offset [Integer]
+          def update_process_state(report, offset)
+            process_name = report[:process][:name]
+
+            state[:processes][process_name] = {
+              dispatched_at: report[:dispatched_at],
+              offset: offset
+            }
+          end
+
+          # Evicts expired processes from the current state
+          # We consider processes dead if they do not report often enough
+          # @note We do not evict based on states (stopped), because we want to report the
+          #   stopped processes for extra time within the ttl limitations. This makes tracking of
+          #   things from UX perspective nicer.
+          def evict_expired_processes
+            max_ttl = float_now - ::Karafka::Web.config.ttl / 1_000
+
+            state[:processes].delete_if do |_name, details|
+              details[:dispatched_at] < max_ttl
+            end
+
+            @active_reports.delete_if do |_name, details|
+              details[:dispatched_at] < max_ttl
+            end
+          end
+
+          # Refreshes the counters that are computed based on incoming reports and not a total sum.
+          # For this we use active reports we have in memory. It may not be accurate for the first
+          # few seconds but it is much more optimal from performance perspective than computing
+          # this fetching all data from Kafka for each view.
+          def refresh_current_stats
+            stats = state[:stats]
+
+            stats[:busy] = 0
+            stats[:enqueued] = 0
+            stats[:threads_count] = 0
+            stats[:processes] = 0
+            stats[:rss] = 0
+            stats[:listeners_count] = 0
+            utilization = 0
+
+            @active_reports
+              .values
+              .reject { |report| report[:process][:status] == 'stopped' }
+              .each do |report|
+                report_stats = report[:stats]
+                report_process = report[:process]
+
+                stats[:busy] += report_stats[:busy]
+                stats[:enqueued] += report_stats[:enqueued]
+                stats[:threads_count] += report_process[:concurrency]
+                stats[:processes] += 1
+                stats[:rss] += report_process[:memory_usage]
+                stats[:listeners_count] += report_process[:listeners]
+                utilization += report_stats[:utilization]
+              end
+
+            stats[:utilization] = utilization / (stats[:processes] + 0.0001)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/processing/consumers/state.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Processing
+      module Consumers
+        # Fetches the current consumer processes aggregated state
+        class State
+          extend ::Karafka::Core::Helpers::Time
+
+          class << self
+            # Try bootstrapping from the current state from Kafka if exists and if not, just use
+            # a blank state. Blank state will not be flushed because materialization into Kafka
+            # happens only after first report is received.
+            #
+            # @return [Hash, false] last (current) aggregated processes state or false if no
+            #   state is available
+            def current
+              state_message = ::Karafka::Admin.read_topic(
+                Karafka::Web.config.topics.consumers.states,
+                0,
+                1
+              ).last
+
+              state_message ? state_message.payload : { processes: {}, stats: {} }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/base_contract.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      # Base for all the metric related contracts
+      class BaseContract < ::Karafka::Core::Contractable::Contract
+        class << self
+          # This layer is not for users extensive feedback, thus we can easily use the minimum
+          # error messaging there is.
+          def configure
+            super do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka::Web.gem_root, 'config', 'locales', 'errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('web')
+            end
+          end
+        end
+
+        # @param data [Hash] data for validation
+        # @return [Boolean] true if all good
+        # @raise [Errors::ContractError] invalid report
+        def validate!(data)
+          super(data, Errors::Tracking::ContractError)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/consumers/contracts/consumer_group.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Consumers
+        # Consumer tracking related contracts
+        module Contracts
+          # Expected data for each consumer group
+          # It's mostly about topics details
+          class ConsumerGroup < BaseContract
+            configure
+
+            required(:id) { |val| val.is_a?(String) && !val.empty? }
+            required(:topics) { |val| val.is_a?(Hash) }
+
+            virtual do |data, errors|
+              next unless errors.empty?
+
+              topic_contract = Topic.new
+
+              data.fetch(:topics).each do |_topic_name, details|
+                topic_contract.validate!(details)
+              end
+
+              nil
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/consumers/contracts/job.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Consumers
+        module Contracts
+          # Contract for the job reporting details
+          class Job < BaseContract
+            configure
+
+            required(:consumer) { |val| val.is_a?(String) }
+            required(:consumer_group) { |val| val.is_a?(String) }
+            required(:started_at) { |val| val.is_a?(Float) && val >= 0 }
+            required(:topic) { |val| val.is_a?(String) }
+            required(:partition) { |val| val.is_a?(Integer) && val >= 0 }
+            required(:first_offset) { |val| val.is_a?(Integer) && val >= 0 }
+            required(:last_offset) { |val| val.is_a?(Integer) && val >= 0 }
+            required(:comitted_offset) { |val| val.is_a?(Integer) }
+            required(:type) { |val| %w[consume revoked shutdown].include?(val) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/consumers/contracts/partition.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Consumers
+        module Contracts
+          # Partition metrics required for web to operate
+          class Partition < BaseContract
+            configure
+
+            required(:id) { |val| val.is_a?(Integer) && val >= 0 }
+            required(:lag_stored) { |val| val.is_a?(Integer) }
+            required(:lag_stored_d) { |val| val.is_a?(Integer) }
+            required(:committed_offset) { |val| val.is_a?(Integer) }
+            required(:stored_offset) { |val| val.is_a?(Integer) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/consumers/contracts/report.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Consumers
+        module Contracts
+          # Main consumer process related reporting schema
+          #
+          # Any outgoing reporting needs to match this format for it to work with the statuses
+          # consumer.
+          class Report < BaseContract
+            configure
+
+            required(:schema_version) { |val| val.is_a?(String) }
+            required(:dispatched_at) { |val| val.is_a?(Numeric) && val.positive? }
+            # We have consumers and producer reports and need to ensure that each is handled
+            # in an expected fashion
+            required(:type) { |val| val == 'consumer' }
+
+            nested(:process) do
+              required(:started_at) { |val| val.is_a?(Numeric) && val.positive? }
+              required(:name) { |val| val.is_a?(String) && val.count(':') >= 2 }
+              required(:memory_usage) { |val| val.is_a?(Integer) && val >= 0 }
+              required(:memory_total_usage) { |val| val.is_a?(Integer) && val >= 0 }
+              required(:memory_size) { |val| val.is_a?(Integer) && val >= 0 }
+              required(:status) { |val| ::Karafka::Status::STATES.key?(val.to_sym) }
+              required(:listeners) { |val| val.is_a?(Integer) && val >= 0 }
+              required(:concurrency) { |val| val.is_a?(Integer) && val.positive? }
+
+              required(:cpu_usage) do |val|
+                val.is_a?(Array) &&
+                  val.all? { |key| key.is_a?(Numeric) } &&
+                  val.all? { |key| key >= -1 } &&
+                  val.size == 3
+              end
+            end
+
+            nested(:versions) do
+              required(:karafka) { |val| val.is_a?(String) && !val.empty? }
+              required(:waterdrop) { |val| val.is_a?(String) && !val.empty? }
+              required(:ruby) { |val| val.is_a?(String) && !val.empty? }
+            end
+
+            nested(:stats) do
+              required(:busy) { |val| val.is_a?(Integer) && val >= 0 }
+              required(:enqueued) { |val| val.is_a?(Integer) && val >= 0 }
+              required(:utilization) { |val| val.is_a?(Numeric) && val >= 0 }
+
+              nested(:total) do
+                required(:batches) { |val| val.is_a?(Numeric) && val >= 0 }
+                required(:messages) { |val| val.is_a?(Numeric) && val >= 0 }
+                required(:errors) { |val| val.is_a?(Numeric) && val >= 0 }
+                required(:retries) { |val| val.is_a?(Numeric) && val >= 0 }
+                required(:dead) { |val| val.is_a?(Numeric) && val >= 0 }
+              end
+            end
+
+            # Consumer groups have topics that have partitions
+            required(:consumer_groups) { |val| val.is_a?(Hash) }
+
+            required(:jobs) { |val| val.is_a?(Array) }
+
+            # Validates that all the data about given consumer group is as expected
+            virtual do |data, errors|
+              next unless errors.empty?
+
+              cg_contract = ConsumerGroup.new
+
+              # Consumer group id (key) is irrelevant because it is also in the details
+              data.fetch(:consumer_groups).each do |_, details|
+                cg_contract.validate!(details)
+              end
+
+              nil
+            end
+
+            # Validates that job reference has all the needed info
+            virtual do |data, errors|
+              next unless errors.empty?
+
+              job_contract = Job.new
+
+              data.fetch(:jobs).each do |details|
+                job_contract.validate!(details)
+              end
+
+              nil
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/consumers/contracts/topic.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Consumers
+        module Contracts
+          # Expected topic information that needs to go out
+          class Topic < BaseContract
+            required(:name) { |val| val.is_a?(String) && !val.empty? }
+            required(:partitions) { |val| val.is_a?(Hash) }
+
+            virtual do |data, errors|
+              next unless errors.empty?
+
+              partition_contract = Partition.new
+
+              data.fetch(:partitions).each do |_partition_id, details|
+                partition_contract.validate!(details)
+              end
+
+              nil
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/consumers/listeners/base.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Consumers
+        # Consumer monitoring related listeners
+        module Listeners
+          # Base consumers processes related listener
+          class Base
+            include ::Karafka::Core::Helpers::Time
+            extend Forwardable
+
+            def_delegators :sampler, :track
+            def_delegators :reporter, :report, :report!
+
+            private
+
+            # @return [Object] sampler in use
+            def sampler
+              @sampler ||= ::Karafka::Web.config.tracking.consumers.sampler
+            end
+
+            # @return [Object] reported in use
+            def reporter
+              @reporter ||= ::Karafka::Web.config.tracking.reporter
+            end
+          end
+        end
+      end
+    end
+  end
+end