karafka-web 0.6.3 → 0.7.0

data/lib/karafka/web/config.rb

@@ -23,6 +23,9 @@ module Karafka
 
           # Topic for storing states aggregated info
           setting :states, default: 'karafka_consumers_states'
+
+          # Topic for storing consumers historical metrics info
+          setting :metrics, default: 'karafka_consumers_metrics'
         end
       end
 
@@ -71,21 +74,42 @@ module Karafka
         # What should be the consumer group name for web consumer
         setting :consumer_group, default: 'karafka_web'
 
-        # How often should we report the aggregated state
-        setting :interval, default: 1_000
-
-        setting :consumers do
-          setting :aggregator, default: Processing::Consumers::Aggregator.new
-        end
+        # How often should we report the aggregated state and metrics
+        # By default we flush the states twice as often as the data reporting.
+        # This will allow us to have closer to real-time reporting.
+        setting :interval, default: 2_500
       end
 
       setting :ui do
-        # Should the payload be decrypted for the Pro Web UI. Default to `false` due to security
-        # reasons
-        setting :decrypt, default: false
+        # UI session settings
+        # Should be set per ENV.
+        setting :sessions do
+          # Cookie key name
+          setting :key, default: '_karafka_session'
+
+          # Secret for the session cookie
+          setting :secret, default: SecureRandom.hex(32)
+        end
+
+        # UI cache to improve performance of views that reuse states that are not often changed
+        setting :cache, default: Ui::Lib::TtlCache.new(60_000 * 5)
+
+        # Should we display internal topics of Kafka. The once starting with `__`
+        # By default we do not display them as they are not usable from regular users perspective
+        setting :show_internal_topics, default: false
 
         # How many elements should we display on pages that support pagination
         setting :per_page, default: 25
+
+        # Time beyond which the last stable offset freeze is considered a risk
+        # (unless same as high). This is used to show on the UI that there may be a hanging
+        # transaction that will cause given consumer group to halt processing and wait
+        setting :lso_threshold, default: 5 * 60 * 1_000
+
+        # Allows to manage visibility of payload, headers and message key in the UI
+        # In some cases you may want to limit what is being displayed due to the type of data you
+        # are dealing with
+        setting :visibility_filter, default: Ui::Models::VisibilityFilter.new
       end
     end
   end

data/lib/karafka/web/contracts/base.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    # Namespace for contracts across the web
+    module Contracts
+      # Base for all the contracts
+      class Base < ::Karafka::Core::Contractable::Contract
+        class << self
+          # This layer is not for users extensive feedback, thus we can easily use the minimum
+          # error messaging there is.
+          def configure
+            super do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka::Web.gem_root, 'config', 'locales', 'errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('web')
+            end
+          end
+        end
+
+        # @param data [Hash] data for validation
+        # @return [Boolean] true if all good
+        # @raise [Errors::ContractError] invalid report
+        def validate!(data)
+          super(data, Errors::ContractError)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/contracts/config.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Contracts
+      # Contract to validate Web-UI configuration
+      class Config < Web::Contracts::Base
+        configure
+
+        # Use the same regexp as Karafka for topics validation
+        TOPIC_REGEXP = ::Karafka::Contracts::TOPIC_REGEXP
+
+        required(:ttl) { |val| val.is_a?(Numeric) && val.positive? }
+
+        nested(:topics) do
+          required(:errors) { |val| val.is_a?(String) && TOPIC_REGEXP.match?(val) }
+
+          nested(:consumers) do
+            required(:reports) { |val| val.is_a?(String) && TOPIC_REGEXP.match?(val) }
+            required(:states) { |val| val.is_a?(String) && TOPIC_REGEXP.match?(val) }
+            required(:metrics) { |val| val.is_a?(String) && TOPIC_REGEXP.match?(val) }
+          end
+        end
+
+        nested(:tracking) do
+          # Do not report more often then every second, this could overload the system
+          required(:interval) { |val| val.is_a?(Integer) && val >= 1_000 }
+
+          nested(:consumers) do
+            required(:reporter) { |val| !val.nil? }
+            required(:sampler) { |val| !val.nil? }
+            required(:listeners) { |val| val.is_a?(Array) }
+          end
+
+          nested(:producers) do
+            required(:reporter) { |val| !val.nil? }
+            required(:sampler) { |val| !val.nil? }
+            required(:listeners) { |val| val.is_a?(Array) }
+          end
+        end
+
+        nested(:processing) do
+          required(:active) { |val| [true, false].include?(val) }
+          required(:consumer_group) { |val| val.is_a?(String) && TOPIC_REGEXP.match?(val) }
+          # Do not update data more often not to overload and not to generate too much data
+          required(:interval) { |val| val.is_a?(Integer) && val >= 1_000 }
+        end
+
+        nested(:ui) do
+          nested(:sessions) do
+            required(:key) { |val| val.is_a?(String) && !val.empty? }
+            required(:secret) { |val| val.is_a?(String) && val.length >= 64 }
+          end
+
+          required(:show_internal_topics) { |val| [true, false].include?(val) }
+          required(:cache) { |val| !val.nil? }
+          required(:per_page) { |val| val.is_a?(Integer) && val >= 1 && val <= 100 }
+          required(:visibility_filter) { |val| !val.nil? }
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/deserializer.rb

@@ -6,12 +6,21 @@ module Karafka
     #
     # @note We use `symbolize_names` because we want to use the same convention of hash building
     #   for producing, consuming and displaying metrics related data
+    #
+    # @note We have to check if we compress the data, because older Web-UI versions were not
+    #   compressing the payload.
     class Deserializer
       # @param message [::Karafka::Messages::Message]
       # @return [Object] deserialized data
       def call(message)
+        raw_payload = if message.headers.key?('zlib')
+                        Zlib::Inflate.inflate(message.raw_payload)
+                      else
+                        message.raw_payload
+                      end
+
         ::JSON.parse(
-          message.raw_payload,
+          raw_payload,
           symbolize_names: true
         )
       end

data/lib/karafka/web/errors.rb

@@ -4,21 +4,43 @@ module Karafka
   module Web
     # Karafka::Web related errors
     module Errors
-      # Tracking related errors
-      module Tracking
-        # Raised when the a report is not valid for any reason
-        # This should never happen and if you see this, please open an issue.
-        ContractError = Class.new(::Karafka::Errors::BaseError)
+      # Base class for all errors related to the web ui
+      BaseError = Class.new(::Karafka::Errors::BaseError)
+
+      # Raised when the a report is not valid for any reason
+      # This should never happen and if you see this, please open an issue.
+      ContractError = Class.new(BaseError)
+
+      # Processing related errors namespace
+      module Processing
+        # Raised when we try to process reports but we do not have the current state bootstrapped
+        # If you see this error, it probably means, that you did not bootstrap Web-UI correctly
+        MissingConsumersStateError = Class.new(BaseError)
+
+        # Similar to the above. It should be created during install
+        MissingConsumersMetricsError = Class.new(BaseError)
+
+        # This error occurs when consumer running older version of the web-ui tries to materialize
+        # states from newer versions. Karafka Web-UI provides only backwards compatibility, so
+        # you need to have an up-to-date consumer materializing reported states.
+        #
+        # If you see this error, please make sure that the consumer process that is materializing
+        # your states is running at least the same version as the consumers that are reporting
+        # the states
+        #
+        # If you see this error do not worry. When you get a consumer with up-to-date version,
+        # all the historical metrics will catch up.
+        IncompatibleSchemaError = Class.new(BaseError)
       end
 
       # Ui related errors
       module Ui
         # Raised when we cannot display a given view.
         # This may mean, that request topic was not present or partition or a message.
-        NotFoundError = Class.new(::Karafka::Errors::BaseError)
+        NotFoundError = Class.new(BaseError)
 
         # Raised whe a given feature is available for Pro but not pro used
-        ProOnlyError = Class.new(::Karafka::Errors::BaseError)
+        ProOnlyError = Class.new(BaseError)
       end
     end
   end

data/lib/karafka/web/installer.rb

@@ -4,165 +4,75 @@ module Karafka
   module Web
     # Responsible for setup of the Web UI and Karafka Web-UI related components initialization.
     class Installer
+      include ::Karafka::Helpers::Colorize
+
       # Creates needed topics and the initial zero state, so even if no `karafka server` processes
-      # are running, we can still display the empty UI
+      # are running, we can still display the empty UI. Also adds needed code to the `karafka.rb`
+      # file.
       #
       # @param replication_factor [Integer] replication factor we want to use (1 by default)
-      def bootstrap!(replication_factor: 1)
-        bootstrap_topics!(replication_factor)
-        bootstrap_consumers_state!
-      end
-
-      # Removes all the Karafka topics and creates them again with the same replication factor
-      def reset!
-        states_topic = ::Karafka::Web.config.topics.consumers.states
-        replication_factor = ::Karafka::Admin
-                             .cluster_info
-                             .topics
-                             .find { |topic| topic[:topic_name] == states_topic }
-                             .fetch(:partitions)
-                             .first
-                             .fetch(:replica_count)
-
-        uninstall!
-        bootstrap!(replication_factor: replication_factor)
+      def install(replication_factor: 1)
+        puts
+        puts 'Installing Karafka Web UI...'
+        puts
+        puts 'Creating necessary topics and populating state data...'
+        puts
+        Management::CreateTopics.new.call(replication_factor)
+        puts
+        Management::CreateInitialStates.new.call
+        puts
+        Management::ExtendBootFile.new.call
+        puts
+        puts("Installation #{green('completed')}. Have fun!")
+        puts
       end
 
-      # Removes all the Karafka Web topics
-      def uninstall!
-        [
-          ::Karafka::Web.config.topics.consumers.states,
-          ::Karafka::Web.config.topics.consumers.reports,
-          ::Karafka::Web.config.topics.errors
-        ].each { |topic_name| ::Karafka::Admin.delete_topic(topic_name) }
+      # Creates missing topics and missing zero states. Needs to run for each environment where we
+      # want to use Web-UI
+      #
+      # @param replication_factor [Integer] replication factor we want to use (1 by default)
+      def migrate(replication_factor: 1)
+        puts
+        puts 'Creating necessary topics and populating state data...'
+        puts
+        Management::CreateTopics.new.call(replication_factor)
+        Management::CreateInitialStates.new.call
+        puts
+        puts("Migration #{green('completed')}. Have fun!")
+        puts
       end
 
-      # Adds the extra needed consumer group, topics and routes for Web UI to be able to operate
-      def enable!
-        ::Karafka::App.routes.draw do
-          web_deserializer = ::Karafka::Web::Deserializer.new
-
-          consumer_group ::Karafka::Web.config.processing.consumer_group do
-            # Topic we listen on to materialize the states
-            topic ::Karafka::Web.config.topics.consumers.reports do
-              config(active: false)
-              active ::Karafka::Web.config.processing.active
-              # Since we materialize state in intervals, we can poll for half of this time without
-              # impacting the reporting responsiveness
-              max_wait_time ::Karafka::Web.config.processing.interval / 2
-              max_messages 1_000
-              consumer ::Karafka::Web::Processing::Consumer
-              deserializer web_deserializer
-              manual_offset_management true
-            end
-
-            # We define those two here without consumption, so Web understands how to deserialize
-            # them when used / viewed
-            topic ::Karafka::Web.config.topics.consumers.states do
-              config(active: false)
-              active false
-              deserializer web_deserializer
-            end
-
-            topic ::Karafka::Web.config.topics.errors do
-              config(active: false)
-              active false
-              deserializer web_deserializer
-            end
-          end
-        end
-
-        # Installs all the consumer related listeners
-        ::Karafka::Web.config.tracking.consumers.listeners.each do |listener|
-          ::Karafka.monitor.subscribe(listener)
-        end
-
-        # Installs all the producer related listeners
-        ::Karafka::Web.config.tracking.producers.listeners.each do |listener|
-          ::Karafka.producer.monitor.subscribe(listener)
-        end
+      # Removes all the Karafka topics and creates them again with the same replication factor
+      # @param replication_factor [Integer] replication factor we want to use (1 by default)
+      def reset(replication_factor: 1)
+        puts
+        puts 'Resetting Karafka Web UI...'
+        puts
+        Management::DeleteTopics.new.call
+        puts
+        Management::CreateTopics.new.call(replication_factor)
+        puts
+        Management::CreateInitialStates.new.call
+        puts
+        puts("Resetting #{green('completed')}. Have fun!")
+        puts
       end
 
-      private
-
-      # Creates all the needed topics for the admin UI to work
-      #
-      # @param replication_factor [Integer]
-      def bootstrap_topics!(replication_factor = 1)
-        existing_topics = ::Karafka::Admin.cluster_info.topics.map { |topic| topic[:topic_name] }
-
-        consumers_states_topic = ::Karafka::Web.config.topics.consumers.states
-        consumers_reports_topic = ::Karafka::Web.config.topics.consumers.reports
-        errors_topic = ::Karafka::Web.config.topics.errors
-
-        # Create only if needed
-        unless existing_topics.include?(consumers_states_topic)
-          # This topic needs to have one partition
-          ::Karafka::Admin.create_topic(
-            consumers_states_topic,
-            1,
-            replication_factor,
-            # We care only about the most recent state, previous are irrelevant. So we can easily
-            # compact after one minute. We do not use this beyond the most recent collective
-            # state, hence it all can easily go away.
-            {
-              'cleanup.policy': 'compact',
-              'retention.ms': 60 * 60 * 1_000
-            }
-          )
-        end
-
-        unless existing_topics.include?(consumers_reports_topic)
-          # This topic needs to have one partition
-          ::Karafka::Admin.create_topic(
-            consumers_reports_topic,
-            1,
-            replication_factor,
-            # We do not need to to store this data for longer than 7 days as this data is only used
-            # to materialize the end states
-            # On the other hand we do not want to have it really short-living because in case of a
-            # consumer crash, we may want to use this info to catch up and backfill the state
-            { 'retention.ms': 7 * 24 * 60 * 60 * 1_000 }
-          )
-        end
-
-        unless existing_topics.include?(errors_topic)
-          # All the errors will be dispatched here
-          # This topic can have multiple partitions but we go with one by default. A single Ruby
-          # process should not crash that often and if there is an expectation of a higher volume
-          # of errors, this can be changed by the end user
-          ::Karafka::Admin.create_topic(
-            errors_topic,
-            1,
-            replication_factor,
-            # Remove really old errors (older than 3 months just to preserve space)
-            { 'retention.ms': 3 * 31 * 24 * 60 * 60 * 1_000 }
-          )
-        end
+      # Removes all the Karafka Web topics and cleans after itself.
+      def uninstall
+        puts
+        puts 'Uninstalling Karafka Web UI...'
+        puts
+        Management::DeleteTopics.new.call
+        Management::CleanBootFile.new.call
+        puts
+        puts("Uninstalling #{green('completed')}. Goodbye!")
+        puts
       end
 
-      # Creates the initial state record with all values being empty
-      def bootstrap_consumers_state!
-        ::Karafka.producer.produce_sync(
-          topic: Karafka::Web.config.topics.consumers.states,
-          key: Karafka::Web.config.topics.consumers.states,
-          payload: {
-            processes: {},
-            stats: {
-              batches: 0,
-              messages: 0,
-              retries: 0,
-              dead: 0,
-              busy: 0,
-              enqueued: 0,
-              threads_count: 0,
-              processes: 0,
-              rss: 0,
-              listeners_count: 0,
-              utilization: 0
-            }
-          }.to_json
-        )
+      # Enables the Web-UI in the karafka app. Sets up needed routes and listeners.
+      def enable!
+        Management::Enable.new.call
       end
     end
   end

data/lib/karafka/web/management/base.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    # Namespace for all the commands related to management of the Web-UI in the context of Karafka
+    # It includes things like installing, creating needed topics, etc.
+    module Management
+      # Base class for all the commands that we use to manage
+      class Base
+        include ::Karafka::Helpers::Colorize
+
+        private
+
+        # @return [String] green colored word "successfully"
+        def successfully
+          green('successfully')
+        end
+
+        # @return [String] green colored word "already"
+        def already
+          green('already')
+        end
+
+        # @return [Array<String>] topics available in the cluster
+        def existing_topics_names
+          @existing_topics_names ||= ::Karafka::Admin
+                                     .cluster_info
+                                     .topics
+                                     .map { |topic| topic[:topic_name] }
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/clean_boot_file.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      # Cleans the boot file from Karafka Web-UI details.
+      class CleanBootFile < Base
+        # Web-UI enabled code
+        ENABLER_CODE = ExtendBootFile::ENABLER_CODE
+
+        private_constant :ENABLER_CODE
+
+        # Removes the Web-UI boot file data
+        def call
+          karafka_rb = File.readlines(Karafka.boot_file)
+
+          if karafka_rb.any? { |line| line.include?(ENABLER_CODE) }
+            puts 'Updating the Karafka boot file...'
+            karafka_rb.delete_if { |line| line.include?(ENABLER_CODE) }
+
+            File.write(Karafka.boot_file, karafka_rb.join)
+            puts "Karafka boot file #{successfully} updated."
+            puts 'Make sure to remove configuration and other customizations as well.'
+          else
+            puts 'Karafka Web UI components not found in the boot file.'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/create_initial_states.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      # Creates the records needed for the Web-UI to operate.
+      class CreateInitialStates < Base
+        # Defaults stats state that we create in Kafka
+        DEFAULT_STATS = {
+          batches: 0,
+          messages: 0,
+          retries: 0,
+          dead: 0,
+          busy: 0,
+          enqueued: 0,
+          processing: 0,
+          workers: 0,
+          processes: 0,
+          rss: 0,
+          listeners: 0,
+          utilization: 0,
+          lag_stored: 0,
+          errors: 0
+        }.freeze
+
+        # Default empty historicals for first record in Kafka
+        DEFAULT_AGGREGATED = Processing::TimeSeriesTracker::TIME_RANGES
+                             .keys
+                             .map { |range| [range, []] }
+                             .to_h
+                             .freeze
+
+        # WHole default empty state (aside from dispatch time)
+        DEFAULT_STATE = {
+          processes: {},
+          stats: DEFAULT_STATS,
+          schema_state: 'accepted',
+          schema_version: Processing::Consumers::Aggregators::State::SCHEMA_VERSION,
+          dispatched_at: Time.now.to_f
+        }.freeze
+
+        # Default metrics state
+        DEFAULT_METRICS = {
+          aggregated: DEFAULT_AGGREGATED,
+          consumer_groups: DEFAULT_AGGREGATED,
+          dispatched_at: Time.now.to_f,
+          schema_version: Processing::Consumers::Aggregators::Metrics::SCHEMA_VERSION
+        }.freeze
+
+        private_constant :DEFAULT_STATS, :DEFAULT_AGGREGATED
+
+        # Creates the initial states for the Web-UI if needed (if they don't exist)
+        def call
+          if Ui::Models::ConsumersState.current
+            exists('consumers state')
+          else
+            creating('consumers state')
+            ::Karafka.producer.produce_sync(
+              topic: Karafka::Web.config.topics.consumers.states,
+              key: Karafka::Web.config.topics.consumers.states,
+              payload: DEFAULT_STATE.to_json
+            )
+            created('consumers state')
+          end
+
+          if Ui::Models::ConsumersMetrics.current
+            exists('consumers metrics')
+          else
+            creating('consumers metrics')
+            ::Karafka.producer.produce_sync(
+              topic: Karafka::Web.config.topics.consumers.metrics,
+              key: Karafka::Web.config.topics.consumers.metrics,
+              payload: DEFAULT_METRICS.merge(dispatched_at: Time.now.to_f).to_json
+            )
+            created('consumers metrics')
+          end
+        end
+
+        private
+
+        # @param type [String] type of state
+        # @return [String] exists message
+        def exists(type)
+          puts "Initial #{type} #{already} exists."
+        end
+
+        # @param type [String] type of state
+        # @return [String] message that the state is being created
+        def creating(type)
+          puts "Creating #{type} initial record..."
+        end
+
+        # @param type [String] type of state
+        # @return [String] message that the state was created
+        def created(type)
+          puts "Initial #{type} record #{successfully} created."
+        end
+      end
+    end
+  end
+end