karafka-web 0.8.2 → 0.9.0.rc1

data/lib/karafka/web/{ui/pro → pro/ui}/app.rb

@@ -13,17 +13,18 @@
 
 module Karafka
   module Web
-    module Ui
+    # Namespace for all the pro components of the Web UI
+    module Pro
       # Pro Web UI components
-      module Pro
+      module Ui
         # Main Roda Web App that servers all the metrics and stats
-        class App < Ui::Base
-          opts[:root] = Karafka::Web.gem_root.join('lib/karafka/web/ui/pro')
+        class App < Web::Ui::Base
+          opts[:root] = Karafka::Web.gem_root.join('lib/karafka/web/pro/ui')
 
           instance_exec(&CONTEXT_DETAILS)
 
           plugin :render, escape: true, engine: 'erb', allowed_paths: [
-            Karafka::Web.gem_root.join('lib/karafka/web/ui/pro/views'),
+            Karafka::Web.gem_root.join('lib/karafka/web/pro/ui/views'),
             Karafka::Web.gem_root.join('lib/karafka/web/ui/views')
           ]
 
@@ -42,12 +43,25 @@ module Karafka
 
             r.get 'dashboard' do
               @breadcrumbs = false
-              controller = Controllers::Dashboard.new(params)
+              controller = Controllers::DashboardController.new(params)
               controller.index
             end
 
             r.on 'consumers' do
-              controller = Controllers::Consumers.new(params)
+              controller = Controllers::ConsumersController.new(params)
+
+              r.get 'overview' do
+                @breadcrumbs = false
+                controller.index
+              end
+
+              r.get 'performance' do
+                controller.performance
+              end
+
+              r.get 'controls' do
+                controller.controls
+              end
 
               r.on String, 'jobs' do |process_id|
                 r.get 'running' do
@@ -69,14 +83,53 @@ module Karafka
                 controller.details(process_id)
               end
 
+              r.redirect root_path('consumers/overview')
+            end
+
+            r.on 'commands' do
+              controller = Controllers::CommandsController.new(params)
+
+              r.on Integer do |offset_id|
+                controller.show(offset_id)
+              end
+
+              r.get 'recent' do
+                controller.recent
+              end
+
               r.get do
-                @breadcrumbs = false
                 controller.index
               end
             end
 
+            r.on 'commanding' do
+              controller = Controllers::CommandingController.new(params)
+
+              r.post 'quiet_all' do
+                controller.quiet_all
+              end
+
+              r.post 'stop_all' do
+                controller.stop_all
+              end
+
+              r.on String do |process_id|
+                r.post 'probe' do
+                  controller.probe(process_id)
+                end
+
+                r.post 'quiet' do
+                  controller.quiet(process_id)
+                end
+
+                r.post 'stop' do
+                  controller.stop(process_id)
+                end
+              end
+            end
+
             r.on 'jobs' do
-              controller = Controllers::Jobs.new(params)
+              controller = Controllers::JobsController.new(params)
 
               r.get 'running' do
                 controller.running
@@ -90,7 +143,7 @@ module Karafka
             end
 
             r.on 'routing' do
-              controller = Controllers::Routing.new(params)
+              controller = Controllers::RoutingController.new(params)
 
               r.get String do |topic_id|
                 controller.show(topic_id)
@@ -102,7 +155,7 @@ module Karafka
             end
 
             r.on 'explorer' do
-              controller = Controllers::Explorer.new(params)
+              controller = Controllers::ExplorerController.new(params)
 
               r.get String, Integer, 'recent' do |topic_id, partition_id|
                 controller.recent(topic_id, partition_id)
@@ -145,7 +198,7 @@ module Karafka
             end
 
             r.on 'messages' do
-              controller = Controllers::Messages.new(params)
+              controller = Controllers::MessagesController.new(params)
 
               r.post String, Integer, Integer, 'republish' do |topic_id, partition_id, offset|
                 controller.republish(topic_id, partition_id, offset)
@@ -161,12 +214,16 @@ module Karafka
             end
 
             r.on 'health' do
-              controller = Controllers::Health.new(params)
+              controller = Controllers::HealthController.new(params)
 
               r.get 'lags' do
                 controller.lags
               end
 
+              r.get 'cluster_lags' do
+                controller.cluster_lags
+              end
+
               r.get 'offsets' do
                 controller.offsets
               end
@@ -185,21 +242,45 @@ module Karafka
             end
 
             r.on 'cluster' do
-              controller = Controllers::Cluster.new(params)
+              controller = Controllers::ClusterController.new(params)
 
-              r.get 'brokers' do
-                controller.brokers
+              r.get 'replication' do
+                # We use the non-pro controller here because this action is the same
+                controller = Ui::Controllers::ClusterController.new(params)
+                controller.replication
               end
 
-              r.get 'topics' do
-                controller.topics
+              r.get String do |broker_id|
+                controller.show(broker_id)
               end
 
-              r.redirect root_path('cluster/brokers')
+              r.get do
+                controller.index
+              end
+            end
+
+            r.on 'topics' do
+              controller = Controllers::TopicsController.new(params)
+
+              r.get 'config', String do |topic_name|
+                controller.config(topic_name)
+              end
+
+              r.get 'replication', String do |topic_name|
+                controller.replication(topic_name)
+              end
+
+              r.get 'distribution', String do |topic_name|
+                controller.distribution(topic_name)
+              end
+
+              r.get do
+                controller.index
+              end
             end
 
             r.on 'errors' do
-              controller = Controllers::Errors.new(params)
+              controller = Controllers::ErrorsController.new(params)
 
               r.get Integer, Integer do |partition_id, offset|
                 if params.current_offset != -1
@@ -219,12 +300,12 @@ module Karafka
             end
 
             r.get 'dlq' do
-              controller = Controllers::Dlq.new(params)
+              controller = Controllers::DlqController.new(params)
               controller.index
             end
 
             r.get 'status' do
-              controller = Controllers::Status.new(params)
+              controller = Controllers::StatusController.new(params)
               controller.show
             end
           end

data/lib/karafka/web/{ui/pro/controllers/cluster.rb → pro/ui/controllers/base_controller.rb}

@@ -13,13 +13,12 @@
 
 module Karafka
   module Web
-    module Ui
-      module Pro
+    module Pro
+      module Ui
         # Namespace for Pro controllers
         module Controllers
-          # Cluster controller
-          class Cluster < Ui::Controllers::Cluster
-            self.sortable_attributes = Ui::Controllers::Cluster.sortable_attributes
+          # Base Pro controller
+          class BaseController < Web::Ui::Controllers::BaseController
           end
         end
       end

data/lib/karafka/web/pro/ui/controllers/cluster_controller.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        # Namespace for Pro controllers
+        module Controllers
+          # Cluster details controller
+          class ClusterController < Web::Ui::Controllers::ClusterController
+            self.sortable_attributes = %w[
+              id
+              name
+              default?
+              read_only?
+              synonym?
+              sensitive?
+              port
+            ].freeze
+
+            # Lists available brokers in the cluster
+            def index
+              @brokers = refine(Models::Broker.all)
+
+              render
+            end
+
+            # Displays selected broker configuration
+            #
+            # @param broker_id [String] id of the broker
+            def show(broker_id)
+              @broker = Models::Broker.find(broker_id)
+
+              @configs = refine(@broker.configs)
+
+              render
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/controllers/commanding_controller.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Controllers
+          # Controller responsible for handling requests that should trigger some action on the
+          # consumers.
+          class CommandingController < BaseController
+            # Dispatches the probe request to a given process
+            #
+            # @param process_id [String]
+            def probe(process_id)
+              command(:probe, process_id)
+
+              redirect(
+                :back,
+                success: dispatched_to_one_msg(:probe, process_id)
+              )
+            end
+
+            # Dispatches the quiet request to a given process
+            #
+            # @param process_id [String]
+            def quiet(process_id)
+              command(:quiet, process_id)
+
+              redirect(
+                :back,
+                success: dispatched_to_one_msg(:quiet, process_id)
+              )
+            end
+
+            # Dispatches the stop request to a given process
+            #
+            # @param process_id [String]
+            def stop(process_id)
+              command(:stop, process_id)
+
+              redirect(
+                :back,
+                success: dispatched_to_one_msg(:stop, process_id)
+              )
+            end
+
+            # Dispatches the quiet request that should trigger on all consumers
+            def quiet_all
+              command(:quiet, '*')
+
+              redirect(
+                :back,
+                success: dispatched_to_all(:quiet)
+              )
+            end
+
+            # Dispatches the stop request that should trigger on all consumers
+            def stop_all
+              command(:stop, '*')
+
+              redirect(
+                :back,
+                success: dispatched_to_all(:stop)
+              )
+            end
+
+            private
+
+            # Dispatches given command
+            # @param command [Symbol] command
+            # @param process_id [String] process id
+            def command(command, process_id)
+              Commanding::Dispatcher.command(command, process_id)
+            end
+
+            # @param process_id [String] find given process
+            def find_process(process_id)
+              Models::Process.find(
+                Models::ConsumersState.current!,
+                process_id
+              )
+            end
+
+            # Generates a nice flash message about the dispatch
+            # @param command [Symbol]
+            # @param process_id [String]
+            # @return [String] flash message that command has been dispatched to a given process
+            def dispatched_to_one_msg(command, process_id)
+              command_name = command.to_s.capitalize
+
+              "The #{command_name} command has been dispatched to the #{process_id} process."
+            end
+
+            # Generates a nice flash message about dispatch of multi-process command
+            # @param command [Symbol]
+            # @return [String] flash message that command has been dispatched
+            def dispatched_to_all(command)
+              command_name = command.to_s.capitalize
+
+              "The #{command_name} command has been dispatched to all active processes."
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/controllers/commands_controller.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Controllers
+          # Controller for viewing details of dispatched commands and their results
+          class CommandsController < BaseController
+            # Lists commands from the consumers commands topic
+            def index
+              @schema_version = Commanding::Dispatcher::SCHEMA_VERSION
+              @watermark_offsets = Models::WatermarkOffsets.find(commands_topic, 0)
+
+              previous_offset, @command_messages, next_offset = current_partition_data
+
+              # If message is an array, it means it's a compacted dummy offset representation
+              mapped_messages = @command_messages.map do |message|
+                message.is_a?(Array) ? message.last : message.offset
+              end
+
+              paginate(
+                previous_offset,
+                @params.current_offset,
+                next_offset,
+                mapped_messages
+              )
+
+              render
+            end
+
+            # Shows details about given command / result
+            #
+            # @param offset [Integer] offset of the command message we're interested in
+            def show(offset)
+              @schema_version = Commanding::Dispatcher::SCHEMA_VERSION
+              @command_message = Models::Message.find(
+                commands_topic,
+                0,
+                offset
+              )
+
+              render
+            end
+
+            # Displays the most recent available command message details
+            def recent
+              # We read 25 just in case there would be a lot of transactional noise
+              messages = ::Karafka::Admin.read_topic(commands_topic, 0, 25)
+
+              # We find the most recent (last since they are shipped in reverse order)
+              recent = messages.reverse.find { |message| !message.is_a?(Array) }
+
+              return show(recent.offset) if recent
+
+              # If nothing found, lets redirect user back to the commands list
+              redirect(
+                'commands',
+                warning: 'No recent commands available.'
+              )
+            end
+
+            private
+
+            # @return [Array] Array with requested messages as well as pagination details and other
+            #   obtained metadata
+            def current_partition_data
+              Models::Message.offset_page(
+                commands_topic,
+                0,
+                @params.current_offset,
+                @watermark_offsets
+              )
+            end
+
+            # @return [String] consumers commands topic name
+            def commands_topic
+              ::Karafka::Web.config.topics.consumers.commands
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/{ui/pro/controllers/consumers.rb → pro/ui/controllers/consumers_controller.rb}

@@ -13,14 +13,18 @@
 
 module Karafka
   module Web
-    module Ui
-      module Pro
+    module Pro
+      module Ui
         module Controllers
           # Controller for displaying consumers states and details about them
-          class Consumers < Ui::Controllers::Base
+          class ConsumersController < BaseController
+            include Web::Ui::Lib::Paginations
+
             self.sortable_attributes = %w[
               id
+              process_id
               name
+              status
               started_at
               lag_hybrid
               lag_hybrid_d
@@ -36,6 +40,15 @@ module Karafka
               first_offset
               last_offset
               updated_at
+              memory_usage
+              threads
+              utilization
+              busy
+              workers
+              active
+              standby
+              running_jobs_count
+              pending_jobs_count
             ].freeze
 
             # Consumers list
@@ -43,7 +56,7 @@ module Karafka
               @current_state = Models::ConsumersState.current!
               @counters = Models::Counters.new(@current_state)
 
-              @processes, last_page = Lib::Paginations::Paginators::Arrays.call(
+              @processes, last_page = Paginators::Arrays.call(
                 refine(Models::Processes.active(@current_state)),
                 @params.current_page
               )
@@ -53,6 +66,20 @@ module Karafka
               render
             end
 
+            # Displays per-process performance details
+            def performance
+              index
+
+              render
+            end
+
+            # Displays processes with details + ability to manage
+            def controls
+              index
+
+              render
+            end
+
             # @param process_id [String] id of the process we're interested in
             def details(process_id)
               current_state = Models::ConsumersState.current!

data/lib/karafka/web/{ui/pro/controllers/dashboard.rb → pro/ui/controllers/dashboard_controller.rb}

@@ -13,11 +13,13 @@
 
 module Karafka
   module Web
-    module Ui
-      module Pro
+    module Pro
+      module Ui
         module Controllers
           # Main Karafka Pro Web-Ui dashboard controller
-          class Dashboard < Ui::Controllers::Base
+          class DashboardController < BaseController
+            include Web::Ui
+
             # View with statistics dashboard details
             def index
               @current_state = Models::ConsumersState.current!

data/lib/karafka/web/pro/ui/controllers/dlq_controller.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Controllers
+          # DLQ topics overview
+          class DlqController < BaseController
+            # Lists DLQ topics
+            def index
+              topics = Karafka::App.consumer_groups.flat_map(&:topics).flat_map(&:to_a)
+
+              dlq_topic_patterns = topics
+                                   .map { |source_topic| source_topic.dead_letter_queue.topic }
+                                   .uniq
+                                   .compact
+                                   .select(&:itself)
+
+              dlq_topic_patterns += Web.config.ui.dlq_patterns
+
+              @dlq_topics = Models::ClusterInfo
+                            .topics
+                            .select { |topic| dlq?(dlq_topic_patterns, topic[:topic_name]) }
+                            .sort_by { |topic| topic[:topic_name] }
+
+              render
+            end
+
+            private
+
+            # Checks if topic is in topics we consider DLQ or if it matches any predefined regular
+            # expressions used for auto-DLQ discovery
+            #
+            # @param topics_matches [Array<String, Regexp>] array with list of DLQ topics matches
+            #   from the routing and regular expressions for auto-discovery
+            # @param name [String] topic name for which we want to know if it is used in DLQ
+            # @return [Boolean] is the given topic a DLQ topic
+            def dlq?(topics_matches, name)
+              topics_matches.any? do |candidate|
+                candidate.match?(name)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/{ui/pro/controllers/errors.rb → pro/ui/controllers/errors_controller.rb}

@@ -13,13 +13,11 @@
 
 module Karafka
   module Web
-    module Ui
-      module Pro
+    module Pro
+      module Ui
         module Controllers
           # Errors details controller
-          class Errors < Ui::Controllers::Base
-            include Ui::Lib::Paginations
-
+          class ErrorsController < BaseController
             # Lists all the errors from all the partitions
             def index
               @topic_id = errors_topic
@@ -41,7 +39,7 @@ module Karafka
             # @param partition_id [Integer] id of the partition of errors we are interested in
             def partition(partition_id)
               @partition_id = partition_id
-              @watermark_offsets = Ui::Models::WatermarkOffsets.find(errors_topic, @partition_id)
+              @watermark_offsets = Models::WatermarkOffsets.find(errors_topic, @partition_id)
               @partitions_count = Models::ClusterInfo.partitions_count(errors_topic)
 
               previous_offset, @error_messages, next_offset = Models::Message.offset_page(
@@ -74,7 +72,7 @@ module Karafka
                 @offset
               )
 
-              watermark_offsets = Ui::Models::WatermarkOffsets.find(errors_topic, partition_id)
+              watermark_offsets = Models::WatermarkOffsets.find(errors_topic, partition_id)
               paginate(offset, watermark_offsets.low, watermark_offsets.high)
 
               render