karafka-web 0.10.4 → 0.11.0.beta1

data/lib/karafka/web/pro/ui/controllers/cluster_controller.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Web

data/lib/karafka/web/pro/ui/controllers/consumers/base_controller.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Controllers
+          # Namespace for all controllers related to consumers operations
+          module Consumers
+            # Base controller for all consumer related controllers
+            class BaseController < Controllers::BaseController
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/controllers/consumers/commanding_controller.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Controllers
+          module Consumers
+            # Controller responsible for handling requests that should trigger some action on the
+            # consumers.
+            class CommandingController < BaseController
+              # Dispatches the trace request to a given process
+              #
+              # @param process_id [String]
+              def trace(process_id)
+                features.commanding!
+
+                request(
+                  Commanding::Commands::Consumers::Trace,
+                  process_id
+                )
+
+                redirect(
+                  :back,
+                  success: dispatched_to_one(:trace, process_id)
+                )
+              end
+
+              # Dispatches the quiet request to a given process
+              #
+              # @param process_id [String]
+              def quiet(process_id)
+                features.commanding!
+
+                request(
+                  Commanding::Commands::Consumers::Quiet,
+                  process_id
+                )
+
+                redirect(
+                  :back,
+                  success: dispatched_to_one(:quiet, process_id)
+                )
+              end
+
+              # Dispatches the stop request to a given process
+              #
+              # @param process_id [String]
+              def stop(process_id)
+                features.commanding!
+
+                request(
+                  Commanding::Commands::Consumers::Stop,
+                  process_id
+                )
+
+                redirect(
+                  :back,
+                  success: dispatched_to_one(:stop, process_id)
+                )
+              end
+
+              # Dispatches the quiet request that should trigger on all consumers
+              def quiet_all
+                features.commanding!
+
+                request(
+                  Commanding::Commands::Consumers::Quiet,
+                  '*'
+                )
+
+                redirect(
+                  :back,
+                  success: dispatched_to_all(:quiet)
+                )
+              end
+
+              # Dispatches the stop request that should trigger on all consumers
+              def stop_all
+                features.commanding!
+
+                request(
+                  Commanding::Commands::Consumers::Stop,
+                  '*'
+                )
+
+                redirect(
+                  :back,
+                  success: dispatched_to_all(:stop)
+                )
+              end
+
+              private
+
+              # Dispatches given command
+              # @param command [Class] command class
+              # @param process_id [String] process id
+              def request(command, process_id)
+                Commanding::Dispatcher.request(
+                  command.name,
+                  process_id
+                )
+              end
+
+              # @param process_id [String] find given process
+              def find_process(process_id)
+                Models::Process.find(
+                  Models::ConsumersState.current!,
+                  process_id
+                )
+              end
+
+              # Generates a nice flash message about the dispatch
+              # @param name [Symbol]
+              # @param process_id [String]
+              # @return [String] flash message that command has been dispatched to a given process
+              def dispatched_to_one(name, process_id)
+                command_name = name.to_s.capitalize
+
+                format_flash(
+                  'The ? command has been dispatched to the ? process',
+                  command_name,
+                  process_id
+                )
+              end
+
+              # Generates a nice flash message about dispatch of multi-process command
+              # @param name [Symbol]
+              # @return [String] flash message that command has been dispatched
+              def dispatched_to_all(name)
+                command_name = name.to_s.capitalize
+
+                format_flash(
+                  'The ? command has been dispatched to ? active processes',
+                  command_name,
+                  'all'
+                )
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/controllers/consumers/commands_controller.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Controllers
+          module Consumers
+            # Controller for viewing details of dispatched commands and their results
+            class CommandsController < BaseController
+              # Lists commands from the consumers commands topic
+              def index
+                features.commanding!
+
+                @schema_version = Commanding::Dispatcher::SCHEMA_VERSION
+                @watermark_offsets = Models::WatermarkOffsets.find(commands_topic, 0)
+
+                previous_offset, @command_messages, next_offset = current_partition_data
+
+                # If message is an array, it means it's a compacted dummy offset representation
+                mapped_messages = @command_messages.map do |message|
+                  message.is_a?(Array) ? message.last : message.offset
+                end
+
+                paginate(
+                  previous_offset,
+                  @params.current_offset,
+                  next_offset,
+                  mapped_messages
+                )
+
+                render
+              end
+
+              # Shows details about given command / result
+              #
+              # @param offset [Integer] offset of the command message we're interested in
+              def show(offset)
+                features.commanding!
+
+                @schema_version = Commanding::Dispatcher::SCHEMA_VERSION
+                @command_message = Models::Message.find(
+                  commands_topic,
+                  0,
+                  offset
+                )
+
+                render
+              end
+
+              # Displays the most recent available command message details
+              def recent
+                features.commanding!
+
+                # We read 25 just in case there would be a lot of transactional noise
+                messages = ::Karafka::Admin.read_topic(commands_topic, 0, 25)
+
+                # We find the most recent (last since they are shipped in reverse order)
+                recent = messages.reverse.find { |message| !message.is_a?(Array) }
+
+                return show(recent.offset) if recent
+
+                # If nothing found, lets redirect user back to the commands list
+                redirect(
+                  'commands',
+                  warning: 'No recent commands available'
+                )
+              end
+
+              private
+
+              # @return [Array] Array with requested messages as well as pagination details and other
+              #   obtained metadata
+              def current_partition_data
+                Models::Message.offset_page(
+                  commands_topic,
+                  0,
+                  @params.current_offset,
+                  @watermark_offsets
+                )
+              end
+
+              # @return [String] consumers commands topic name
+              def commands_topic
+                ::Karafka::Web.config.topics.consumers.commands.name
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/controllers/consumers/consumers_controller.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Controllers
+          module Consumers
+            # Controller for displaying consumers states and details about them
+            class ConsumersController < BaseController
+              include Web::Ui::Lib::Paginations
+
+              self.sortable_attributes = %w[
+                id
+                process_id
+                name
+                status
+                started_at
+                lag_hybrid
+                lag_hybrid_d
+                committed_offset
+                stored_offset
+                fetch_state
+                poll_state
+                lso_risk_state
+                topic
+                consumer
+                type
+                messages
+                first_offset
+                last_offset
+                updated_at
+                memory_usage
+                threads
+                utilization
+                busy
+                workers
+                active
+                standby
+                running_jobs_count
+                pending_jobs_count
+              ].freeze
+
+              # Consumers list
+              def index
+                @current_state = Models::ConsumersState.current!
+                @counters = Models::Counters.new(@current_state)
+
+                @processes, last_page = Paginators::Arrays.call(
+                  refine(Models::Processes.active(@current_state)),
+                  @params.current_page
+                )
+
+                paginate(@params.current_page, !last_page)
+
+                render
+              end
+
+              # Displays per-process performance details
+              def performance
+                index
+
+                render
+              end
+
+              # @param process_id [String] id of the process we're interested in
+              def details(process_id)
+                current_state = Models::ConsumersState.current!
+                @process = Models::Process.find(current_state, process_id)
+
+                return render if @process.schema_compatible?
+
+                raise Errors::Ui::IncompatibleSchemaError
+              end
+
+              # @param process_id [String] id of the process we're interested in
+              def subscriptions(process_id)
+                details(process_id)
+
+                # We want to have sorting but on a per subscription group basis and not to sort
+                # everything
+                @process.consumer_groups.each do |consumer_group|
+                  # We need to initialize the whole structure so dynamic fields are also built into
+                  # the underlying hashes for sorting
+                  consumer_group.subscription_groups.flat_map(&:topics).flat_map(&:partitions)
+
+                  refine(consumer_group)
+                end
+
+                render
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/controllers/consumers/controls_controller.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Controllers
+          module Consumers
+            # Displays processes with details + ability to manage
+            class ControlsController < ConsumersController
+              self.sortable_attributes = %w[
+                id
+                status
+                started_at
+                memory_usage
+                lag_hybrid
+              ].freeze
+
+              # Displays list of consumer processes + options to manage them
+              def index
+                features.commanding!
+
+                super
+
+                render
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/controllers/consumers/jobs_controller.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Controllers
+          module Consumers
+            # Displays details about given consumer jobs
+            #
+            # @note There is a separate jobs controller for jobs overview, this one is per consumer
+            #   specific.
+            class JobsController < ConsumersController
+              self.sortable_attributes = %w[
+                topic
+                consumer
+                type
+                messages
+                first_offset
+                last_offset
+                committed_offset
+                updated_at
+              ].freeze
+
+              # Shows all running jobs of a consumer
+              # @param process_id [String]
+              def running(process_id)
+                details(process_id)
+
+                @running_jobs = @process.jobs.running
+
+                refine(@running_jobs)
+
+                render
+              end
+
+              # Shows all pending jobs of a consumer
+              # @param process_id [String]
+              def pending(process_id)
+                details(process_id)
+
+                @pending_jobs = @process.jobs.pending
+
+                refine(@pending_jobs)
+
+                render
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/controllers/consumers/partitions/base_controller.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Controllers
+          module Consumers
+            # Namespace for controllers related to working with particular partitions in the
+            # context of consumers processes
+            module Partitions
+              # Base controller for all the partition related management stuff
+              class BaseController < ConsumersController
+                private
+
+                # Finds all the needed details and if not found raises a not found.
+                # This prevents most cases where something would change between visiting the pages
+                # and dispatching no longer valid data.
+                #
+                # @param process_id [String]
+                # @param subscription_group_id [String]
+                # @param topic [String]
+                # @param partition_id [Integer]
+                def bootstrap!(
+                  process_id,
+                  subscription_group_id,
+                  topic,
+                  partition_id
+                )
+                  subscriptions(process_id)
+
+                  @subscription_group_id = subscription_group_id
+                  @topic = topic
+                  @partition_id = partition_id
+                  @consumer_group = nil
+                  @subscription_group = nil
+                  @partition_stats = nil
+
+                  # Looks for the appropriate details about given partition and so on in the
+                  # current process data. Since we operate in the context of the given process,
+                  # it must have those details. If not it means that assignment most likely have
+                  # changed and it is no longer valid anyhow.
+                  @process.consumer_groups.each do |consumer_group|
+                    consumer_group.subscription_groups.each do |subscription_group|
+                      next unless subscription_group.id == @subscription_group_id
+
+                      @subscription_group = subscription_group
+                      @consumer_group = consumer_group
+
+                      subscription_group.topics.each do |s_topic|
+                        next unless s_topic.name == @topic
+
+                        s_topic.partitions.each do |partition|
+                          next unless @partition_id.to_s == partition.partition_id.to_s
+
+                          @partition_stats = partition
+                        end
+                      end
+                    end
+                  end
+
+                  @subscription_group || raise(Karafka::Web::Errors::Ui::NotFoundError)
+
+                  routing_topics = Karafka::App.routes.flat_map(&:topics).flat_map(&:to_a)
+
+                  @routing_topic = routing_topics.find do |r_topic|
+                    r_topic.subscription_group.id == @subscription_group.id &&
+                      r_topic.name == @topic
+                  end
+
+                  @partition_stats || raise(Karafka::Web::Errors::Ui::NotFoundError)
+                  # May not be found when not all routing is available. In such cases we assume
+                  # that topic is not LRJ and it's up to the end user to handle this correctly.
+                  @topic_lrj = @routing_topic&.long_running_job?
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/controllers/consumers/partitions/offsets_controller.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Controllers
+          module Consumers
+            module Partitions
+              # Partition offset management controller in the context of current consumer process
+              # assignments
+              class OffsetsController < BaseController
+                self.sortable_attributes = [].freeze
+
+                # Displays the offset edit page with the edit form or a warning when not applicable
+                #
+                # @param process_id [String] id of the process we're interested in
+                # @param subscription_group_id [String]
+                # @param topic [String]
+                # @param partition_id [Integer]
+                def edit(process_id, subscription_group_id, topic, partition_id)
+                  bootstrap!(process_id, subscription_group_id, topic, partition_id)
+
+                  render
+                end
+
+                # Triggers the offset change in the running process via the commanding API
+                #
+                # @param process_id [String] id of the process we're interested in
+                # @param subscription_group_id [String]
+                # @param topic [String]
+                # @param partition_id [Integer]
+                def update(process_id, subscription_group_id, topic, partition_id)
+                  edit(process_id, subscription_group_id, topic, partition_id)
+
+                  offset = params.int(:offset)
+                  prevent_overtaking = params.bool(:prevent_overtaking)
+                  force_resume = params.bool(:force_resume)
+
+                  Commanding::Dispatcher.request(
+                    Commanding::Commands::Partitions::Seek.name,
+                    process_id,
+                    {
+                      consumer_group_id: @consumer_group.id,
+                      subscription_group_id: @subscription_group.id,
+                      topic: topic,
+                      partition_id: partition_id,
+                      offset: offset,
+                      prevent_overtaking: prevent_overtaking,
+                      force_resume: force_resume
+                    }
+                  )
+
+                  redirect(
+                    :previous,
+                    success: format_flash(
+                      'Initiated offset adjustment to ? for ?#? (subscription group: ?)',
+                      offset,
+                      topic,
+                      partition_id,
+                      subscription_group_id
+                    )
+                  )
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end