karafka-web 0.10.4 → 0.11.0.beta1

data/lib/karafka/web/pro/commanding/commands/stop.rb

@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Web
-    module Pro
-      module Commanding
-        # Namespace for commands the process can react to
-        module Commands
-          # Sends a signal to stop the process
-          # @note Does not work in an embedded mode because we do not own the Ruby process.
-          class Stop < Base
-            # Performs the command if not in embedded mode
-            def call
-              return unless standalone?
-
-              ::Process.kill('QUIT', ::Process.pid)
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/web/pro/commanding/commands/trace.rb

@@ -1,41 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Web
-    module Pro
-      module Commanding
-        # Namespace for commands the process can react to
-        module Commands
-          # Collects all backtraces from the available Ruby threads and publishes their details
-          #   back to Kafka for debug.
-          class Trace < Base
-            # Runs tracing and publishes result back to Kafka
-            def call
-              threads = {}
-
-              Thread.list.each do |thread|
-                tid = (thread.object_id ^ ::Process.pid).to_s(36)
-                t_d = threads[tid] = {}
-                t_d[:label] = "Thread TID-#{tid} #{thread.name}"
-                t_d[:backtrace] = (thread.backtrace || ['<no backtrace available>']).join("\n")
-              end
-
-              Dispatcher.result(threads, process_id, 'trace')
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/web/pro/ui/controllers/commanding_controller.rb

@@ -1,118 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Web
-    module Pro
-      module Ui
-        module Controllers
-          # Controller responsible for handling requests that should trigger some action on the
-          # consumers.
-          class CommandingController < BaseController
-            # Dispatches the trace request to a given process
-            #
-            # @param process_id [String]
-            def trace(process_id)
-              command(:trace, process_id)
-
-              redirect(
-                :back,
-                success: dispatched_to_one(:trace, process_id)
-              )
-            end
-
-            # Dispatches the quiet request to a given process
-            #
-            # @param process_id [String]
-            def quiet(process_id)
-              command(:quiet, process_id)
-
-              redirect(
-                :back,
-                success: dispatched_to_one(:quiet, process_id)
-              )
-            end
-
-            # Dispatches the stop request to a given process
-            #
-            # @param process_id [String]
-            def stop(process_id)
-              command(:stop, process_id)
-
-              redirect(
-                :back,
-                success: dispatched_to_one(:stop, process_id)
-              )
-            end
-
-            # Dispatches the quiet request that should trigger on all consumers
-            def quiet_all
-              command(:quiet, '*')
-
-              redirect(
-                :back,
-                success: dispatched_to_all(:quiet)
-              )
-            end
-
-            # Dispatches the stop request that should trigger on all consumers
-            def stop_all
-              command(:stop, '*')
-
-              redirect(
-                :back,
-                success: dispatched_to_all(:stop)
-              )
-            end
-
-            private
-
-            # Dispatches given command
-            # @param command [Symbol] command
-            # @param process_id [String] process id
-            def command(command, process_id)
-              Commanding::Dispatcher.command(command, process_id)
-            end
-
-            # @param process_id [String] find given process
-            def find_process(process_id)
-              Models::Process.find(
-                Models::ConsumersState.current!,
-                process_id
-              )
-            end
-
-            # Generates a nice flash message about the dispatch
-            # @param command [Symbol]
-            # @param process_id [String]
-            # @return [String] flash message that command has been dispatched to a given process
-            def dispatched_to_one(command, process_id)
-              command_name = command.to_s.capitalize
-
-              "The #{command_name} command has been dispatched to the #{process_id} process."
-            end
-
-            # Generates a nice flash message about dispatch of multi-process command
-            # @param command [Symbol]
-            # @return [String] flash message that command has been dispatched
-            def dispatched_to_all(command)
-              command_name = command.to_s.capitalize
-
-              "The #{command_name} command has been dispatched to all active processes."
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/web/pro/ui/controllers/commands_controller.rb

@@ -1,96 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Web
-    module Pro
-      module Ui
-        module Controllers
-          # Controller for viewing details of dispatched commands and their results
-          class CommandsController < BaseController
-            # Lists commands from the consumers commands topic
-            def index
-              @schema_version = Commanding::Dispatcher::SCHEMA_VERSION
-              @watermark_offsets = Models::WatermarkOffsets.find(commands_topic, 0)
-
-              previous_offset, @command_messages, next_offset = current_partition_data
-
-              # If message is an array, it means it's a compacted dummy offset representation
-              mapped_messages = @command_messages.map do |message|
-                message.is_a?(Array) ? message.last : message.offset
-              end
-
-              paginate(
-                previous_offset,
-                @params.current_offset,
-                next_offset,
-                mapped_messages
-              )
-
-              render
-            end
-
-            # Shows details about given command / result
-            #
-            # @param offset [Integer] offset of the command message we're interested in
-            def show(offset)
-              @schema_version = Commanding::Dispatcher::SCHEMA_VERSION
-              @command_message = Models::Message.find(
-                commands_topic,
-                0,
-                offset
-              )
-
-              render
-            end
-
-            # Displays the most recent available command message details
-            def recent
-              # We read 25 just in case there would be a lot of transactional noise
-              messages = ::Karafka::Admin.read_topic(commands_topic, 0, 25)
-
-              # We find the most recent (last since they are shipped in reverse order)
-              recent = messages.reverse.find { |message| !message.is_a?(Array) }
-
-              return show(recent.offset) if recent
-
-              # If nothing found, lets redirect user back to the commands list
-              redirect(
-                'commands',
-                warning: 'No recent commands available.'
-              )
-            end
-
-            private
-
-            # @return [Array] Array with requested messages as well as pagination details and other
-            #   obtained metadata
-            def current_partition_data
-              Models::Message.offset_page(
-                commands_topic,
-                0,
-                @params.current_offset,
-                @watermark_offsets
-              )
-            end
-
-            # @return [String] consumers commands topic name
-            def commands_topic
-              ::Karafka::Web.config.topics.consumers.commands
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/web/pro/ui/controllers/consumers_controller.rb

@@ -1,138 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Web
-    module Pro
-      module Ui
-        module Controllers
-          # Controller for displaying consumers states and details about them
-          class ConsumersController < BaseController
-            include Web::Ui::Lib::Paginations
-
-            self.sortable_attributes = %w[
-              id
-              process_id
-              name
-              status
-              started_at
-              lag_hybrid
-              lag_hybrid_d
-              committed_offset
-              stored_offset
-              fetch_state
-              poll_state
-              lso_risk_state
-              topic
-              consumer
-              type
-              messages
-              first_offset
-              last_offset
-              updated_at
-              memory_usage
-              threads
-              utilization
-              busy
-              workers
-              active
-              standby
-              running_jobs_count
-              pending_jobs_count
-            ].freeze
-
-            # Consumers list
-            def index
-              @current_state = Models::ConsumersState.current!
-              @counters = Models::Counters.new(@current_state)
-
-              @processes, last_page = Paginators::Arrays.call(
-                refine(Models::Processes.active(@current_state)),
-                @params.current_page
-              )
-
-              paginate(@params.current_page, !last_page)
-
-              render
-            end
-
-            # Displays per-process performance details
-            def performance
-              index
-
-              render
-            end
-
-            # Displays processes with details + ability to manage
-            def controls
-              index
-
-              render
-            end
-
-            # @param process_id [String] id of the process we're interested in
-            def details(process_id)
-              current_state = Models::ConsumersState.current!
-              @process = Models::Process.find(current_state, process_id)
-
-              render
-            end
-
-            # Renders details about running jobs
-            #
-            # @param process_id [String] id of the process we're interested in
-            def running_jobs(process_id)
-              details(process_id)
-
-              @running_jobs = @process.jobs.running
-
-              refine(@running_jobs)
-
-              render
-            end
-
-            # Renders details about pending jobs
-            #
-            # @param process_id [String] id of the process we're interested in
-            def pending_jobs(process_id)
-              details(process_id)
-
-              @pending_jobs = @process.jobs.pending
-
-              refine(@pending_jobs)
-
-              render
-            end
-
-            # @param process_id [String] id of the process we're interested in
-            def subscriptions(process_id)
-              details(process_id)
-
-              # We want to have sorting but on a per subscription group basis and not to sort
-              # everything
-              @process.consumer_groups.each do |consumer_group|
-                # We need to initialize the whole structure so dynamic fields are also built into
-                # the underlying hashes for sorting
-                consumer_group.subscription_groups.flat_map(&:topics).flat_map(&:partitions)
-
-                refine(consumer_group)
-              end
-
-              render
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/web/pro/ui/controllers/explorer_controller.rb

@@ -1,220 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Web
-    module Pro
-      module Ui
-        module Controllers
-          # Data explorer controller
-          class ExplorerController < BaseController
-            # Lists all the topics we can explore
-            def index
-              @topics = Models::ClusterInfo
-                        .topics
-                        .sort_by { |topic| topic[:topic_name] }
-
-              unless ::Karafka::Web.config.ui.visibility.internal_topics
-                @topics.reject! { |topic| topic[:topic_name].start_with?('__') }
-              end
-
-              render
-            end
-
-            # Displays aggregated messages from (potentially) all partitions of a topic
-            #
-            # @param topic_id [String]
-            #
-            # @note This view may not be 100% accurate because we merge multiple partitions data
-            #   into a single view and this is never accurate. It can be used however to quickly
-            #   look at most recent data flowing, etc, hence it is still useful for aggregated
-            #   metrics information
-            #
-            # @note We cannot use offset references here because each of the partitions may have
-            #   completely different values
-            def topic(topic_id)
-              @visibility_filter = ::Karafka::Web.config.ui.policies.messages
-
-              @topic_id = topic_id
-              @partitions_count = Models::ClusterInfo.partitions_count(topic_id)
-
-              @active_partitions, materialized_page, @limited = Paginators::Partitions.call(
-                @partitions_count, @params.current_page
-              )
-
-              @messages, next_page = Models::Message.topic_page(
-                topic_id, @active_partitions, materialized_page
-              )
-
-              paginate(@params.current_page, next_page)
-
-              render
-            end
-
-            # Shows messages available in a given partition
-            #
-            # @param topic_id [String]
-            # @param partition_id [Integer]
-            def partition(topic_id, partition_id)
-              @visibility_filter = ::Karafka::Web.config.ui.policies.messages
-              @topic_id = topic_id
-              @partition_id = partition_id
-              @watermark_offsets = Models::WatermarkOffsets.find(topic_id, partition_id)
-              @partitions_count = Models::ClusterInfo.partitions_count(topic_id)
-
-              previous_offset, @messages, next_offset = current_partition_data
-
-              paginate(
-                previous_offset,
-                @params.current_offset,
-                next_offset,
-                # If message is an array, it means it's a compacted dummy offset representation
-                @messages.map { |message| message.is_a?(Array) ? message.last : message.offset }
-              )
-
-              render
-            end
-
-            # Displays given message
-            #
-            # @param topic_id [String]
-            # @param partition_id [Integer]
-            # @param offset [Integer] offset of the message we want to display
-            # @param paginate [Boolean] do we want to have pagination
-            def show(topic_id, partition_id, offset, paginate: true)
-              Lib::PatternsDetector.new.call
-
-              @visibility_filter = ::Karafka::Web.config.ui.policies.messages
-              @topic_id = topic_id
-              @partition_id = partition_id
-              @offset = offset
-              @message = Models::Message.find(@topic_id, @partition_id, @offset)
-
-              @safe_key = Web::Pro::Ui::Lib::SafeRunner.new { @message.key }.tap(&:call)
-              @safe_headers = Web::Pro::Ui::Lib::SafeRunner.new { @message.headers }.tap(&:call)
-              @safe_payload = Web::Pro::Ui::Lib::SafeRunner.new { @message.payload }.tap(&:call)
-
-              # This may be off for certain views like recent view where we are interested only
-              # in the most recent all the time. It does not make any sense to display pagination
-              # there
-              if paginate
-                # We need watermark offsets to decide if we can paginate left and right
-                watermark_offsets = Models::WatermarkOffsets.find(topic_id, partition_id)
-                paginate(offset, watermark_offsets.low, watermark_offsets.high)
-              end
-
-              render
-            end
-
-            # Displays the most recent message on a topic/partition
-            #
-            # @param topic_id [String]
-            # @param partition_id [Integer, nil] partition we're interested in or nil if we are
-            #   interested in the most recent message from all the partitions
-            def recent(topic_id, partition_id)
-              if partition_id
-                active_partitions = [partition_id]
-              else
-                partitions_count = Models::ClusterInfo.partitions_count(topic_id)
-                active_partitions, = Paginators::Partitions.call(partitions_count, 1)
-              end
-
-              recent = nil
-
-              # This selects first pages with most recent messages and moves to next if first
-              # contains only compacted data, etc.
-              #
-              # We do it until we find a message we could refer to (if doable) within first
-              # ten pages
-              10.times do |page|
-                messages, = Models::Message.topic_page(topic_id, active_partitions, page + 1)
-
-                # Selects newest out of all partitions
-                # Reject compacted messages and transaction-related once
-                recent = messages.reject { |message| message.is_a?(Array) }.max_by(&:timestamp)
-
-                break if recent
-              end
-
-              recent || not_found!
-
-              show(topic_id, recent.partition, recent.offset, paginate: false)
-            end
-
-            # Computes a page on which the given offset is in the middle of the page (if possible)
-            # Useful often when debugging to be able to quickly jump to the historical location
-            # of message and its surrounding to understand failure
-            #
-            # @param topic_id [String]
-            # @param partition_id [Integer]
-            # @param offset [Integer] offset of the message we want to display
-            def surrounding(topic_id, partition_id, offset)
-              watermark_offsets = Models::WatermarkOffsets.find(topic_id, partition_id)
-
-              not_found! if offset < watermark_offsets.low
-              not_found! if offset >= watermark_offsets.high
-
-              # Assume we start from this offset
-              shift = 0
-              elements = 0
-
-              # Position the offset as close to the middle of offset based page as possible
-              ::Karafka::Web.config.ui.per_page.times do
-                break if elements >= ::Karafka::Web.config.ui.per_page
-
-                elements += 1 if offset + shift < watermark_offsets.high
-
-                if offset - shift > watermark_offsets.low
-                  shift += 1
-                  elements += 1
-                end
-              end
-
-              target = offset - shift
-
-              redirect("explorer/#{topic_id}/#{partition_id}?offset=#{target}")
-            end
-
-            # Finds the closest offset matching the requested time and redirects to this location
-            # Note, that it redirects to closest but always younger.
-            #
-            # @param topic_id [String]
-            # @param partition_id [Integer]
-            # @param time [Time] time of the message
-            def closest(topic_id, partition_id, time)
-              target = Web::Ui::Lib::Admin.read_topic(topic_id, partition_id, 1, time).first
-
-              partition_path = "explorer/#{topic_id}/#{partition_id}"
-              partition_path += "?offset=#{target.offset}" if target
-
-              redirect(partition_path)
-            end
-
-            private
-
-            # Fetches current page data
-            # @return [Array] fetched data with pagination information for the requested partition
-            def current_partition_data
-              Models::Message.offset_page(
-                @topic_id,
-                @partition_id,
-                @params.current_offset,
-                @watermark_offsets
-              )
-            end
-          end
-        end
-      end
-    end
-  end
-end