karafka-web 0.10.4 → 0.11.0.beta1

data/lib/karafka/web/config.rb

@@ -35,24 +35,79 @@ module Karafka
       # Topics naming - used for processing and UI
       setting :topics do
         # All the errors encountered will be dispatched to this topic for inspection
-        setting :errors, default: 'karafka_errors'
+        setting :errors do
+          setting :name, default: 'karafka_errors'
+
+          # Remove really old errors (older than 3 months just to preserve space)
+          setting :config, default: {
+            'cleanup.policy': 'delete',
+            'retention.ms': 3 * 31 * 24 * 60 * 60 * 1_000 # 3 months
+          }
+        end
 
         setting :consumers do
           # Reports containing particular consumer processes. This topic contains the heartbeat
           # information sent from each consumer process.
-          setting :reports, default: 'karafka_consumers_reports'
+          setting :reports do
+            # Name of the topic
+            setting :name, default: 'karafka_consumers_reports'
+
+            # We do not need to to store this data for longer than 1 day as this data is only
+            # used to materialize the end states
+            # On the other hand we do not want to have it really short-living because in case
+            # of a consumer crash, we may want to use this info to catch up and backfill the
+            # state.
+            #
+            # In case its not consumed because no processes are running, it also usually means
+            # there's no data to consume because no karafka servers report
+            setting :config, default: {
+              'cleanup.policy': 'delete',
+              'retention.ms': 24 * 60 * 60 * 1_000 # 1 day
+            }
+          end
 
           # Topic for storing states aggregated info
-          setting :states, default: 'karafka_consumers_states'
+          setting :states do
+            setting :name, default: 'karafka_consumers_states'
+
+            # We care only about the most recent state, previous are irrelevant. So we can
+            # easily compact after one minute. We do not use this beyond the most recent
+            # collective state, hence it all can easily go away. We also limit the segment
+            # size to at most 100MB not to use more space ever.
+            setting :config, default: {
+              'cleanup.policy': 'compact',
+              'retention.ms': 60 * 60 * 1_000,
+              'segment.ms': 24 * 60 * 60 * 1_000, # 1 day
+              'segment.bytes': 104_857_600 # 100MB
+            }
+          end
 
           # Topic for storing consumers historical metrics info
-          setting :metrics, default: 'karafka_consumers_metrics'
+          setting :metrics do
+            setting :name, default: 'karafka_consumers_metrics'
+
+            setting :config, default: {
+              'cleanup.policy': 'compact',
+              'retention.ms': 24 * 60 * 60 * 1_000, # 1 day
+              'segment.ms': 24 * 60 * 60 * 1_000, # 1 day
+              'segment.bytes': 104_857_600 # 100MB
+            }
+          end
 
           # Topic for storing commands and their results
           # This is used only in Pro, however we do setup it in OSS in case of upgrade so the
           # transition from one to another is smooth. Otherwise upgrade would require changes
           # to topics (migration) which may be more complex
-          setting :commands, default: 'karafka_consumers_commands'
+          setting :commands do
+            setting :name, default: 'karafka_consumers_commands'
+
+            setting :config, default: {
+              'cleanup.policy': 'delete',
+              'retention.ms': 7 * 24 * 60 * 60 * 1_000, # 7 days
+              'segment.ms': 24 * 60 * 60 * 1_000, # 1 day
+              'segment.bytes': 104_857_600 # 100MB
+            }
+          end
         end
       end
 
@@ -93,7 +148,8 @@ module Karafka
             Tracking::Consumers::Listeners::Statistics.new,
             Tracking::Consumers::Listeners::Pausing.new,
             Tracking::Consumers::Listeners::Processing.new,
-            Tracking::Consumers::Listeners::Tags.new
+            Tracking::Consumers::Listeners::Tags.new,
+            Tracking::Consumers::Listeners::Transactions.new
           ]
         end
 
@@ -150,10 +206,13 @@ module Karafka
           setting :secret, default: SecureRandom.hex(32)
         end
 
-        # UI cache to improve performance of views that reuse states that are not often changed
-        setting :cache, default: Ui::Lib::TtlCache.new(
+        # Process cache with invalidation only
+        setting :cache, default: Ui::Lib::Cache.new(
           # Use the TTL for internal cache in prod but invalidate quickly in other environments,
-          # as for example in development things may change frequently
+          # as for example in development things may change frequently. In dev also the cache
+          # is less needed since local clusters are usually smaller than prod.
+          # It is set to 5 minutes by default because in prod the cluster chances are not frequent
+          # and anyhow we refresh on certain web ui actions so this is as fail-safe
           Karafka.env.production? ? 60_000 * 5 : 5_000
         )
 
@@ -186,6 +245,18 @@ module Karafka
         # default will not be displayed not to hang the browser. 512KB of serialized data is a lot.
         setting :max_visible_payload_size, default: 524_288
 
+        setting :custom do
+          # Path to a custom CSS, string with the CSS or false if not used
+          setting :css, default: false
+
+          # Path to a custom JS, string with the JS or false if not used
+          setting :js, default: false
+
+          # ERB template (path or string) for injecting extra navigation items below the default
+          # menu. Useful for adding "Back to app" links or other quick access actions.
+          setting :nav_erb, default: false
+        end
+
         # Specific kafka settings that are tuned to operate within the Web UI interface.
         #
         # Please do not change them unless you know what you are doing as their misconfiguration

data/lib/karafka/web/contracts/config.rb

@@ -15,12 +15,46 @@ module Karafka
         required(:group_id) { |val| val.is_a?(String) && TOPIC_REGEXP.match?(val) }
 
         nested(:topics) do
-          required(:errors) { |val| val.is_a?(String) && TOPIC_REGEXP.match?(val) }
+          nested(:errors) do
+            required(:name) { |val| val.is_a?(String) && TOPIC_REGEXP.match?(val) }
+
+            required(:config) do |val|
+              val.is_a?(Hash) && !val.empty? && val.keys.all? { |key| key.is_a?(Symbol) }
+            end
+          end
 
           nested(:consumers) do
-            required(:reports) { |val| val.is_a?(String) && TOPIC_REGEXP.match?(val) }
-            required(:states) { |val| val.is_a?(String) && TOPIC_REGEXP.match?(val) }
-            required(:metrics) { |val| val.is_a?(String) && TOPIC_REGEXP.match?(val) }
+            nested(:reports) do
+              required(:name) { |val| val.is_a?(String) && TOPIC_REGEXP.match?(val) }
+
+              required(:config) do |val|
+                val.is_a?(Hash) && !val.empty? && val.keys.all? { |key| key.is_a?(Symbol) }
+              end
+            end
+
+            nested(:states) do
+              required(:name) { |val| val.is_a?(String) && TOPIC_REGEXP.match?(val) }
+
+              required(:config) do |val|
+                val.is_a?(Hash) && !val.empty? && val.keys.all? { |key| key.is_a?(Symbol) }
+              end
+            end
+
+            nested(:metrics) do
+              required(:name) { |val| val.is_a?(String) && TOPIC_REGEXP.match?(val) }
+
+              required(:config) do |val|
+                val.is_a?(Hash) && !val.empty? && val.keys.all? { |key| key.is_a?(Symbol) }
+              end
+            end
+
+            nested(:commands) do
+              required(:name) { |val| val.is_a?(String) && TOPIC_REGEXP.match?(val) }
+
+              required(:config) do |val|
+                val.is_a?(Hash) && !val.empty? && val.keys.all? { |key| key.is_a?(Symbol) }
+              end
+            end
           end
         end
 
@@ -60,11 +94,16 @@ module Karafka
             required(:secret) { |val| val.is_a?(String) && val.length >= 64 }
           end
 
-          required(:cache) { |val| !val.nil? }
           required(:per_page) { |val| val.is_a?(Integer) && val >= 1 && val <= 100 }
           required(:max_visible_payload_size) { |val| val.is_a?(Integer) && val >= 1 }
           required(:kafka) { |val| val.is_a?(Hash) }
 
+          nested(:custom) do
+            required(:css) { |val| val == false || (val.is_a?(String) && !val.empty?) }
+            required(:js) { |val| val == false || (val.is_a?(String) && !val.empty?) }
+            required(:nav_erb) { |val| val == false || (val.is_a?(String) && !val.empty?) }
+          end
+
           required(:dlq_patterns) do |val|
             val.is_a?(Array) &&
               val.all? { |attr| attr.is_a?(String) || attr.is_a?(Regexp) }

data/lib/karafka/web/errors.rb

@@ -20,6 +20,11 @@ module Karafka
       # configures Karafka
       KarafkaNotInitializedError = Class.new(BaseError)
 
+      # Raised when you try to configure Web UI after it was enabled. It is not allowed because
+      # Karafka Web UI uses the setup values during the enablement and their later change may not
+      # be fully reflected. Always run `#setup` before `#enable!`.
+      LateSetupError = Class.new(BaseError)
+
       # Errors specific to management
       module Management
         # Similar to processing error with the same name, it is raised when a critical
@@ -74,6 +79,11 @@ module Karafka
 
         # Raised when we want to stop the flow and render 403
         ForbiddenError = Class.new(BaseError)
+
+        # Raised when trying to get info about a consumer that has incompatible schema in its
+        # report. It usually means you are running different version of the Web UI in the consumer
+        # and in the Web server
+        IncompatibleSchemaError = Class.new(BaseError)
       end
     end
   end

data/lib/karafka/web/management/actions/create_initial_states.rb

@@ -20,25 +20,25 @@ module Karafka
 
           # Creates the initial states for the Web-UI if needed (if they don't exist)
           def call
-            if exists?(Karafka::Web.config.topics.consumers.states)
+            if exists?(Karafka::Web.config.topics.consumers.states.name)
               exists('consumers state')
             else
               creating('consumers state')
               ::Karafka::Web.producer.produce_sync(
-                topic: Karafka::Web.config.topics.consumers.states,
-                key: Karafka::Web.config.topics.consumers.states,
+                topic: Karafka::Web.config.topics.consumers.states.name,
+                key: Karafka::Web.config.topics.consumers.states.name,
                 payload: DEFAULT_STATE.to_json
               )
               created('consumers state')
             end
 
-            if exists?(Karafka::Web.config.topics.consumers.metrics)
+            if exists?(Karafka::Web.config.topics.consumers.metrics.name)
               exists('consumers metrics')
             else
               creating('consumers metrics')
               ::Karafka::Web.producer.produce_sync(
-                topic: Karafka::Web.config.topics.consumers.metrics,
-                key: Karafka::Web.config.topics.consumers.metrics,
+                topic: Karafka::Web.config.topics.consumers.metrics.name,
+                key: Karafka::Web.config.topics.consumers.metrics.name,
                 payload: DEFAULT_METRICS.to_json
               )
               created('consumers metrics')

data/lib/karafka/web/management/actions/create_topics.rb

@@ -23,114 +23,80 @@ module Karafka
             consumers_commands_topic = ::Karafka::Web.config.topics.consumers.commands
             errors_topic = ::Karafka::Web.config.topics.errors
 
-            if existing_topics_names.include?(errors_topic)
-              exists(errors_topic)
+            if existing_topics_names.include?(errors_topic.name)
+              exists(errors_topic.name)
             else
-              creating(errors_topic)
+              creating(errors_topic.name)
               # All the errors will be dispatched here
               # This topic can have multiple partitions but we go with one by default. A single
               # Ruby process should not crash that often and if there is an expectation of a higher
               # volume of errors, this can be changed by the end user
               ::Karafka::Admin.create_topic(
-                errors_topic,
+                errors_topic.name,
                 1,
                 replication_factor,
-                # Remove really old errors (older than 3 months just to preserve space)
-                {
-                  'cleanup.policy': 'delete',
-                  'retention.ms': 3 * 31 * 24 * 60 * 60 * 1_000 # 3 months
-                }
+                errors_topic.config
               )
-              created(errors_topic)
+              created(errors_topic.name)
             end
 
-            if existing_topics_names.include?(consumers_reports_topic)
-              exists(consumers_reports_topic)
+            if existing_topics_names.include?(consumers_reports_topic.name)
+              exists(consumers_reports_topic.name)
             else
-              creating(consumers_reports_topic)
+              creating(consumers_reports_topic.name)
               # This topic needs to have one partition
               ::Karafka::Admin.create_topic(
-                consumers_reports_topic,
+                consumers_reports_topic.name,
                 1,
                 replication_factor,
-                # We do not need to to store this data for longer than 1 day as this data is only
-                # used to materialize the end states
-                # On the other hand we do not want to have it really short-living because in case
-                # of a consumer crash, we may want to use this info to catch up and backfill the
-                # state.
-                #
-                # In case its not consumed because no processes are running, it also usually means
-                # there's no data to consume because no karafka servers report
-                {
-                  'cleanup.policy': 'delete',
-                  'retention.ms': 24 * 60 * 60 * 1_000 # 1 day
-                }
+                consumers_reports_topic.config
               )
-              created(consumers_reports_topic)
+              created(consumers_reports_topic.name)
             end
 
-            if existing_topics_names.include?(consumers_metrics_topic)
-              exists(consumers_metrics_topic)
+            if existing_topics_names.include?(consumers_metrics_topic.name)
+              exists(consumers_metrics_topic.name)
             else
-              creating(consumers_metrics_topic)
+              creating(consumers_metrics_topic.name)
               # This topic needs to have one partition
               # Same as states - only most recent is relevant as it is a materialized state
               ::Karafka::Admin.create_topic(
-                consumers_metrics_topic,
+                consumers_metrics_topic.name,
                 1,
                 replication_factor,
-                {
-                  'cleanup.policy': 'compact',
-                  'retention.ms': 24 * 60 * 60 * 1_000, # 1 day
-                  'segment.ms': 24 * 60 * 60 * 1_000, # 1 day
-                  'segment.bytes': 104_857_600 # 100MB
-                }
+                consumers_metrics_topic.config
               )
-              created(consumers_metrics_topic)
+              created(consumers_metrics_topic.name)
             end
 
-            if existing_topics_names.include?(consumers_commands_topic)
-              exists(consumers_commands_topic)
+            if existing_topics_names.include?(consumers_commands_topic.name)
+              exists(consumers_commands_topic.name)
             else
-              creating(consumers_commands_topic)
+              creating(consumers_commands_topic.name)
               # Commands are suppose to live short and be used for controlling processes and some
               # debug. Their data can be removed safely fast.
               ::Karafka::Admin.create_topic(
-                consumers_commands_topic,
+                consumers_commands_topic.name,
                 1,
                 replication_factor,
-                {
-                  'cleanup.policy': 'delete',
-                  'retention.ms': 7 * 24 * 60 * 60 * 1_000, # 7 days
-                  'segment.ms': 24 * 60 * 60 * 1_000, # 1 day
-                  'segment.bytes': 104_857_600 # 100MB
-                }
+                consumers_commands_topic.config
               )
-              created(consumers_commands_topic)
+              created(consumers_commands_topic.name)
             end
 
             # Create only if needed
-            if existing_topics_names.include?(consumers_states_topic)
-              exists(consumers_states_topic)
+            if existing_topics_names.include?(consumers_states_topic.name)
+              exists(consumers_states_topic.name)
             else
-              creating(consumers_states_topic)
+              creating(consumers_states_topic.name)
               # This topic needs to have one partition
               ::Karafka::Admin.create_topic(
-                consumers_states_topic,
+                consumers_states_topic.name,
                 1,
                 replication_factor,
-                # We care only about the most recent state, previous are irrelevant. So we can
-                # easily compact after one minute. We do not use this beyond the most recent
-                # collective state, hence it all can easily go away. We also limit the segment
-                # size to at most 100MB not to use more space ever.
-                {
-                  'cleanup.policy': 'compact',
-                  'retention.ms': 60 * 60 * 1_000,
-                  'segment.ms': 24 * 60 * 60 * 1_000, # 1 day
-                  'segment.bytes': 104_857_600 # 100MB
-                }
+                consumers_states_topic.config
               )
-              created(consumers_states_topic)
+              created(consumers_states_topic.name)
             end
           end
 

data/lib/karafka/web/management/actions/delete_topics.rb

@@ -9,11 +9,11 @@ module Karafka
           # Removes the Web-UI topics
           def call
             [
-              ::Karafka::Web.config.topics.consumers.states,
-              ::Karafka::Web.config.topics.consumers.reports,
-              ::Karafka::Web.config.topics.consumers.metrics,
-              ::Karafka::Web.config.topics.consumers.commands,
-              ::Karafka::Web.config.topics.errors
+              ::Karafka::Web.config.topics.consumers.states.name,
+              ::Karafka::Web.config.topics.consumers.reports.name,
+              ::Karafka::Web.config.topics.consumers.metrics.name,
+              ::Karafka::Web.config.topics.consumers.commands.name,
+              ::Karafka::Web.config.topics.errors.name
             ].each do |topic_name|
               if existing_topics_names.include?(topic_name.to_s)
                 puts "Removing #{topic_name}..."

data/lib/karafka/web/management/actions/enable.rb

@@ -55,7 +55,7 @@ module Karafka
 
               consumer_group ::Karafka::Web.config.group_id do
                 # Topic we listen on to materialize the states
-                topic ::Karafka::Web.config.topics.consumers.reports do
+                topic ::Karafka::Web.config.topics.consumers.reports.name do
                   config(active: false)
                   active ::Karafka::Web.config.processing.active
                   # Since we materialize state in intervals, we can poll for half of this time
@@ -80,25 +80,25 @@ module Karafka
 
                 # We define those three here without consumption, so Web understands how to
                 # deserialize them when used / viewed
-                topic ::Karafka::Web.config.topics.consumers.states do
+                topic ::Karafka::Web.config.topics.consumers.states.name do
                   config(active: false)
                   active false
                   deserializers(payload: payload_deserializer)
                 end
 
-                topic ::Karafka::Web.config.topics.consumers.metrics do
+                topic ::Karafka::Web.config.topics.consumers.metrics.name do
                   config(active: false)
                   active false
                   deserializers(payload: payload_deserializer)
                 end
 
-                topic ::Karafka::Web.config.topics.consumers.commands do
+                topic ::Karafka::Web.config.topics.consumers.commands.name do
                   config(active: false)
                   active false
                   deserializers(payload: payload_deserializer)
                 end
 
-                topic ::Karafka::Web.config.topics.errors do
+                topic ::Karafka::Web.config.topics.errors.name do
                   config(active: false)
                   active false
                   deserializers(payload: payload_deserializer)

data/lib/karafka/web/pro/commanding/commands/base.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Web
@@ -19,11 +11,38 @@ module Karafka
         module Commands
           # Base for all the commands
           class Base
+            class << self
+              attr_accessor :name
+            end
+
+            # @return [Hash]
+            attr_reader :command
+
+            # @param command [Command] command details (if any). Some commands may require extra
+            #   details to work. They can be obtained from here.
+            def initialize(command)
+              @command = command
+            end
+
+            # Executes the command after receiving it.
+            def call
+              raise NotImlementedError, 'Please implement in a subclass'
+            end
+
             private
 
-            # @return [String] current process id
-            def process_id
-              @process_id ||= ::Karafka::Web.config.tracking.consumers.sampler.process_id
+            # Dispatches the acceptance message back to Kafka as a confirmation
+            #
+            # @param params [Hash] hash with the acceptance message details
+            def acceptance(params)
+              Dispatcher.acceptance(self.class.name, process_id, params)
+            end
+
+            # Dispatches the result message back to Kafka with execution details
+            #
+            # @param params [Hash] hash with the result message details
+            def result(params)
+              Dispatcher.result(self.class.name, process_id, params)
             end
 
             # @return [Boolean] Is given process to which a command was sent operating in an
@@ -33,6 +52,11 @@ module Karafka
             def standalone?
               Karafka::Server.execution_mode == :standalone
             end
+
+            # @return [String] id of the current consumer process
+            def process_id
+              ::Karafka::Web.config.tracking.consumers.sampler.process_id
+            end
           end
         end
       end

data/lib/karafka/web/pro/commanding/commands/consumers/quiet.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Web
+    module Pro
+      module Commanding
+        # Namespace for commands the process can react to
+        module Commands
+          # Namespace for commands related to consumers themselves
+          module Consumers
+            # Sends a signal to quiet the consumer
+            # @note Does not work in an embedded mode because we do not own the Ruby process.
+            class Quiet < Base
+              self.name = 'consumers.quiet'
+
+              # Performs the command if not in embedded mode
+              def call
+                return unless standalone?
+
+                ::Process.kill('TSTP', ::Process.pid)
+
+                result(status: 'applied')
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/commanding/commands/consumers/stop.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Web
+    module Pro
+      module Commanding
+        # Namespace for commands the process can react to
+        module Commands
+          module Consumers
+            # Sends a signal to stop the process
+            # @note Does not work in an embedded mode because we do not own the Ruby process.
+            class Stop < Base
+              self.name = 'consumers.stop'
+
+              # Performs the command if not in embedded mode
+              def call
+                return unless standalone?
+
+                ::Process.kill('QUIT', ::Process.pid)
+
+                result(status: 'applied')
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/commanding/commands/consumers/trace.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Web
+    module Pro
+      module Commanding
+        # Namespace for commands the process can react to
+        module Commands
+          module Consumers
+            # Collects all backtraces from the available Ruby threads and publishes their details
+            #   back to Kafka for debug.
+            class Trace < Base
+              self.name = 'consumers.trace'
+
+              # Runs tracing and publishes result back to Kafka
+              def call
+                threads = {}
+
+                Thread.list.each do |thread|
+                  tid = (thread.object_id ^ ::Process.pid).to_s(36)
+                  t_d = threads[tid] = {}
+                  t_d[:label] = "Thread TID-#{tid} #{thread.name}"
+                  t_d[:backtrace] = (thread.backtrace || ['<no backtrace available>']).join("\n")
+                end
+
+                result(threads)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/commanding/commands/partitions/pause.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Web
+    module Pro
+      module Commanding
+        module Commands
+          # Namespace for post-fetch command execution
+          module Partitions
+            # Delegates the pause request into the partition changes tracker and dispatches the
+            # acceptance message back to Kafka
+            class Pause < Base
+              self.name = 'partitions.pause'
+
+              # Delegates the pause request to async handling
+              def call
+                Handlers::Partitions::Tracker.instance << command
+
+                acceptance(command.to_h)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end