karafka-web 0.10.4 → 0.11.0.beta1

data/lib/karafka/web/pro/ui/controllers/consumers/partitions/pauses_controller.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Controllers
+          module Consumers
+            module Partitions
+              # Controller for managing partitions pauses in the context of the given consumer
+              # process
+              class PausesController < BaseController
+                self.sortable_attributes = %w[].freeze
+
+                # Displays the toggle (pause / resume)
+                #
+                # @param process_id [String] id of the process we're interested in
+                # @param subscription_group_id [String]
+                # @param topic [String]
+                # @param partition_id [Integer]
+                def new(process_id, subscription_group_id, topic, partition_id)
+                  bootstrap!(process_id, subscription_group_id, topic, partition_id)
+
+                  render
+                end
+
+                # @param process_id [String]
+                # @param subscription_group_id [String]
+                # @param topic [String]
+                # @param partition_id [Integer]
+                def create(process_id, subscription_group_id, topic, partition_id)
+                  new(process_id, subscription_group_id, topic, partition_id)
+
+                  Commanding::Dispatcher.request(
+                    Commanding::Commands::Partitions::Pause.name,
+                    process_id,
+                    {
+                      consumer_group_id: @consumer_group.id,
+                      subscription_group_id: @subscription_group.id,
+                      topic: topic,
+                      partition_id: partition_id,
+                      # User provides this in seconds, we operate on ms in the system
+                      duration: params.int(:duration) * 1_000,
+                      prevent_override: params.bool(:prevent_override)
+                    }
+                  )
+
+                  redirect(
+                    :previous,
+                    success: format_flash(
+                      'Initiated partition ? for ?#? (subscription group: ?)',
+                      'pause',
+                      topic,
+                      partition_id,
+                      subscription_group_id
+                    )
+                  )
+                end
+
+                # @param process_id [String]
+                # @param subscription_group_id [String]
+                # @param topic [String]
+                # @param partition_id [Integer]
+                def edit(process_id, subscription_group_id, topic, partition_id)
+                  new(process_id, subscription_group_id, topic, partition_id)
+
+                  render
+                end
+
+                # @param process_id [String]
+                # @param subscription_group_id [String]
+                # @param topic [String]
+                # @param partition_id [Integer]
+                def delete(process_id, subscription_group_id, topic, partition_id)
+                  new(process_id, subscription_group_id, topic, partition_id)
+
+                  Commanding::Dispatcher.request(
+                    Commanding::Commands::Partitions::Resume.name,
+                    process_id,
+                    {
+                      consumer_group_id: @consumer_group.id,
+                      subscription_group_id: @subscription_group.id,
+                      topic: topic,
+                      partition_id: partition_id,
+                      reset_attempts: params.bool(:reset_attempts)
+                    }
+                  )
+
+                  redirect(
+                    :previous,
+                    success: format_flash(
+                      'Initiated partition ? for ?#? (subscription group: ?)',
+                      'resume',
+                      topic,
+                      partition_id,
+                      subscription_group_id
+                    )
+                  )
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/controllers/dashboard_controller.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Web

data/lib/karafka/web/pro/ui/controllers/dlq_controller.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Web

data/lib/karafka/web/pro/ui/controllers/errors_controller.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Web
@@ -83,7 +75,7 @@ module Karafka
 
             # @return [String] errors topic
             def errors_topic
-              ::Karafka::Web.config.topics.errors
+              ::Karafka::Web.config.topics.errors.name
             end
           end
         end

data/lib/karafka/web/pro/ui/controllers/explorer/base_controller.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Controllers
+          # Namespace for all controllers related to explorer operations
+          module Explorer
+            # Base controller for all explorer related controllers
+            class BaseController < Controllers::BaseController
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/controllers/explorer/explorer_controller.rb

@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Controllers
+          module Explorer
+            # Data explorer controller
+            class ExplorerController < BaseController
+              # Lists all the topics we can explore
+              def index
+                @topics = Models::ClusterInfo
+                          .topics
+                          .sort_by { |topic| topic[:topic_name] }
+
+                unless ::Karafka::Web.config.ui.visibility.internal_topics
+                  @topics.reject! { |topic| topic[:topic_name].start_with?('__') }
+                end
+
+                render
+              end
+
+              # Displays aggregated messages from (potentially) all partitions of a topic
+              #
+              # @param topic_id [String]
+              #
+              # @note This view may not be 100% accurate because we merge multiple partitions data
+              #   into a single view and this is never accurate. It can be used however to quickly
+              #   look at most recent data flowing, etc, hence it is still useful for aggregated
+              #   metrics information
+              #
+              # @note We cannot use offset references here because each of the partitions may have
+              #   completely different values
+              def topic(topic_id)
+                @visibility_filter = ::Karafka::Web.config.ui.policies.messages
+
+                @topic_id = topic_id
+                @partitions_count = Models::ClusterInfo.partitions_count(topic_id)
+
+                @active_partitions, materialized_page, @limited = Paginators::Partitions.call(
+                  @partitions_count, @params.current_page
+                )
+
+                @messages, next_page = Models::Message.topic_page(
+                  topic_id, @active_partitions, materialized_page
+                )
+
+                paginate(@params.current_page, next_page)
+
+                render
+              end
+
+              # Shows messages available in a given partition
+              #
+              # @param topic_id [String]
+              # @param partition_id [Integer]
+              def partition(topic_id, partition_id)
+                @visibility_filter = ::Karafka::Web.config.ui.policies.messages
+                @topic_id = topic_id
+                @partition_id = partition_id
+                @watermark_offsets = Models::WatermarkOffsets.find(topic_id, partition_id)
+                @partitions_count = Models::ClusterInfo.partitions_count(topic_id)
+
+                previous_offset, @messages, next_offset = current_partition_data
+
+                paginate(
+                  previous_offset,
+                  @params.current_offset,
+                  next_offset,
+                  # If message is an array, it means it's a compacted dummy offset representation
+                  @messages.map { |message| message.is_a?(Array) ? message.last : message.offset }
+                )
+
+                render
+              end
+
+              # Displays given message
+              #
+              # @param topic_id [String]
+              # @param partition_id [Integer]
+              # @param offset [Integer] offset of the message we want to display
+              # @param paginate [Boolean] do we want to have pagination
+              def show(topic_id, partition_id, offset, paginate: true)
+                Lib::PatternsDetector.new.call
+
+                @visibility_filter = ::Karafka::Web.config.ui.policies.messages
+                @topic_id = topic_id
+                @partition_id = partition_id
+                @offset = offset
+                @message = Models::Message.find(@topic_id, @partition_id, @offset)
+
+                @safe_key = Web::Pro::Ui::Lib::SafeRunner.new { @message.key }.tap(&:call)
+                @safe_headers = Web::Pro::Ui::Lib::SafeRunner.new { @message.headers }.tap(&:call)
+                @safe_payload = Web::Pro::Ui::Lib::SafeRunner.new { @message.payload }.tap(&:call)
+
+                # This may be off for certain views like recent view where we are interested only
+                # in the most recent all the time. It does not make any sense to display pagination
+                # there
+                if paginate
+                  # We need watermark offsets to decide if we can paginate left and right
+                  watermark_offsets = Models::WatermarkOffsets.find(topic_id, partition_id)
+                  paginate(offset, watermark_offsets.low, watermark_offsets.high)
+                end
+
+                render
+              end
+
+              # Displays the most recent message on a topic/partition
+              #
+              # @param topic_id [String]
+              # @param partition_id [Integer, nil] partition we're interested in or nil if we are
+              #   interested in the most recent message from all the partitions
+              def recent(topic_id, partition_id)
+                if partition_id
+                  active_partitions = [partition_id]
+                else
+                  partitions_count = Models::ClusterInfo.partitions_count(topic_id)
+                  active_partitions, = Paginators::Partitions.call(partitions_count, 1)
+                end
+
+                recent = nil
+
+                # This selects first pages with most recent messages and moves to next if first
+                # contains only compacted data, etc.
+                #
+                # We do it until we find a message we could refer to (if doable) within first
+                # ten pages
+                10.times do |page|
+                  messages, = Models::Message.topic_page(topic_id, active_partitions, page + 1)
+
+                  # Selects newest out of all partitions
+                  # Reject compacted messages and transaction-related once
+                  recent = messages.reject { |message| message.is_a?(Array) }.max_by(&:timestamp)
+
+                  break if recent
+                end
+
+                recent || not_found!
+
+                show(topic_id, recent.partition, recent.offset, paginate: false)
+              end
+
+              # Computes a page on which the given offset is in the middle of the page if possible
+              # Useful often when debugging to be able to quickly jump to the historical location
+              # of message and its surrounding to understand failure
+              #
+              # @param topic_id [String]
+              # @param partition_id [Integer]
+              # @param offset [Integer] offset of the message we want to display
+              def surrounding(topic_id, partition_id, offset)
+                watermark_offsets = Models::WatermarkOffsets.find(topic_id, partition_id)
+
+                not_found! if offset < watermark_offsets.low
+                not_found! if offset >= watermark_offsets.high
+
+                # Assume we start from this offset
+                shift = 0
+                elements = 0
+
+                # Position the offset as close to the middle of offset based page as possible
+                ::Karafka::Web.config.ui.per_page.times do
+                  break if elements >= ::Karafka::Web.config.ui.per_page
+
+                  elements += 1 if offset + shift < watermark_offsets.high
+
+                  if offset - shift > watermark_offsets.low
+                    shift += 1
+                    elements += 1
+                  end
+                end
+
+                target = offset - shift
+
+                redirect("explorer/topics/#{topic_id}/#{partition_id}?offset=#{target}")
+              end
+
+              # Finds the closest offset matching the requested time and redirects to this location
+              # Note, that it redirects to closest but always younger.
+              #
+              # @param topic_id [String]
+              # @param partition_id [Integer]
+              # @param time [Time] time of the message
+              def closest(topic_id, partition_id, time)
+                target = Web::Ui::Lib::Admin.read_topic(topic_id, partition_id, 1, time).first
+
+                partition_path = "explorer/topics/#{topic_id}/#{partition_id}"
+                partition_path += "?offset=#{target.offset}" if target
+
+                redirect(partition_path)
+              end
+
+              private
+
+              # Fetches current page data
+              # @return [Array] fetched data with pagination information for the requested
+              #   partition
+              def current_partition_data
+                Models::Message.offset_page(
+                  @topic_id,
+                  @partition_id,
+                  @params.current_offset,
+                  @watermark_offsets
+                )
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/controllers/explorer/messages_controller.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Controllers
+          module Explorer
+            # Controller for working with messages
+            # While part of messages operations is done via explorer (exploring), this controller
+            # handles other cases not related to viewing data
+            class MessagesController < BaseController
+              # Renders a form allowing for piping a message to a different topic
+              #
+              # @param topic_id [String]
+              # @param partition_id [Integer]
+              # @param offset [Integer] offset of the message we want to republish
+              def forward(topic_id, partition_id, offset)
+                @message = Models::Message.find(topic_id, partition_id, offset)
+
+                deny! unless visibility_filter.republish?(@message)
+
+                @topic_id = topic_id
+                @partition_id = partition_id
+                @offset = offset
+
+                @target_topic = @topic_id
+                @target_partition = @partition_id
+
+                @topics = Models::ClusterInfo
+                          .topics
+                          .sort_by { |topic| topic[:topic_name] }
+
+                unless ::Karafka::Web.config.ui.visibility.internal_topics
+                  @topics.reject! { |topic| topic[:topic_name].start_with?('__') }
+                end
+
+                render
+              end
+
+              # Takes a requested message content and republishes it again
+              #
+              # @param topic_id [String]
+              # @param partition_id [Integer]
+              # @param offset [Integer] offset of the message we want to republish
+              def republish(topic_id, partition_id, offset)
+                forward(topic_id, partition_id, offset)
+
+                dispatch_message = {
+                  topic: params.str(:target_topic),
+                  payload: @message.raw_payload,
+                  headers: @message.headers.dup,
+                  key: @message.key
+                }
+
+                # Add target partition only if it was requested, otherwise it will use either the
+                # message key (if present) or will jut round-robin
+                unless params.fetch(:target_partition).empty?
+                  dispatch_message[:partition] = params.int(:target_partition)
+                end
+
+                # Include source headers for enhanced debuggability
+                if params.bool(:include_source_headers)
+                  dispatch_message[:headers].merge!(
+                    'source_topic' => @message.topic,
+                    'source_partition' => @message.partition.to_s,
+                    'source_offset' => @message.offset.to_s
+                  )
+                end
+
+                delivery = ::Karafka::Web.producer.produce_sync(dispatch_message)
+
+                redirect(
+                  :previous,
+                  success: republished(@message, delivery)
+                )
+              end
+
+              # Dispatches the message raw payload to the browser as a file
+              #
+              # @param topic_id [String]
+              # @param partition_id [Integer]
+              # @param offset [Integer] offset of the message we want to download
+              def download(topic_id, partition_id, offset)
+                message = Models::Message.find(topic_id, partition_id, offset)
+
+                deny! unless visibility_filter.download?(message)
+
+                file(
+                  message.raw_payload,
+                  "#{topic_id}_#{partition_id}_#{offset}_payload.msg"
+                )
+              end
+
+              # Dispatches the message payload first deserialized and then serialized to JSON
+              # It differs from the raw payload in cases where raw payload is compressed or binary
+              # or contains data that the Web UI user should not see that was altered on the Web UI
+              # with the visibility filter.
+              #
+              # @param topic_id [String]
+              # @param partition_id [Integer]
+              # @param offset [Integer] offset of the message we want to export
+              def export(topic_id, partition_id, offset)
+                Lib::PatternsDetector.new.call
+
+                message = Models::Message.find(topic_id, partition_id, offset)
+
+                # Check if exports are allowed
+                deny! unless visibility_filter.export?(message)
+
+                file(
+                  message.payload.to_json,
+                  "#{topic_id}_#{partition_id}_#{offset}_payload.json"
+                )
+              end
+
+              private
+
+              # @param message [Karafka::Messages::Message]
+              # @param delivery [Rdkafka::Producer::DeliveryReport]
+              # @return [String] flash message about message reproducing
+              def republished(message, delivery)
+                format_flash(
+                  'Message with offset ? has been sent to ?#? and received offset ?',
+                  message.offset,
+                  delivery.topic,
+                  delivery.partition,
+                  delivery.offset
+                )
+              end
+
+              # @return [Object] visibility filter. Either default or user-based
+              def visibility_filter
+                ::Karafka::Web.config.ui.policies.messages
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/controllers/explorer/search_controller.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Controllers
+          module Explorer
+            # Handles the search requests
+            # We present this as a part of explorer scope but we use a separate controller not to
+            # mix data exploring with searching.
+            class SearchController < Web::Ui::Controllers::ClusterController
+              # Runs the search if search parameters are provided
+              # If no parameters provided, displays the search modal and info to provide search
+              # data. If invalid search parameters provided, modal contains errors
+              #
+              # @param topic_id [String] topic we're interested in
+              # @note In theory search can be used to detect pieces of information within messages.
+              #   Since we allow for custom search strategies, this is not an issue because users
+              #   that need to provide only granular search can do so.
+              def index(topic_id)
+                @topic_id = topic_id
+                @partitions_count = Models::ClusterInfo.partitions_count(topic_id)
+                # Select only matchers that should be available in the context of the current topic
+                available_matchers = Web.config.ui.search.matchers
+                @matchers = available_matchers.select { |match| match.active?(@topic_id) }
+                @search_criteria = !@params.current_search.empty?
+                @current_search = Lib::Search::Normalizer.call(@params.current_search)
+                # Needed when rendering found messages rows. We should always filter the messages
+                # details with the visibility filter
+                @visibility_filter = ::Karafka::Web.config.ui.policies.messages
+                @limits = ::Karafka::Web.config.ui.search.limits.sort
+
+                # If there is search form filled, we validate it to make sure there are no errors
+                @errors = if @search_criteria
+                            Lib::Search::Contracts::Form.new.call(@current_search).errors
+                          else
+                            {}
+                          end
+
+                # If all good we run the search
+                if @search_criteria && @errors.empty?
+                  found, @search_details = Lib::Search::Runner.new(
+                    @topic_id,
+                    @partitions_count,
+                    @current_search
+                  ).call
+
+                  @messages, last_page = Paginators::Arrays.call(
+                    found,
+                    @params.current_page
+                  )
+
+                  paginate(@params.current_page, !last_page)
+                end
+
+                render
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/controllers/health_controller.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Web

data/lib/karafka/web/pro/ui/controllers/jobs_controller.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Web