karafka-web 0.9.2 → 0.10.0.beta1

data/lib/karafka/web/pro/ui/app.rb

@@ -32,6 +32,19 @@ module Karafka
             Karafka::Web.gem_root.join('lib/karafka/web/ui/views')
           ]
 
+          before do
+            assets_path = root_path("assets/#{Karafka::Web::VERSION}/")
+
+            # Always allow assets
+            break true if request.path.start_with?(assets_path)
+            # If policies extension is not loaded, allow as this is the default
+            break true unless Web.config.ui.respond_to?(:policies)
+            break true if Web.config.ui.policies.requests.allow?(env)
+
+            # Do not allow if given request violates requests policies
+            raise(Errors::Ui::ForbiddenError)
+          end
+
           route do |r|
             r.root { r.redirect root_path('dashboard') }
 
@@ -51,7 +64,6 @@ module Karafka
               controller = Controllers::ConsumersController.new(params)
 
               r.get 'overview' do
-                @breadcrumbs = false
                 controller.index
               end
 
@@ -155,6 +167,13 @@ module Karafka
             end
 
             r.on 'explorer' do
+              r.get String, 'search' do |topic_id|
+                # Search has it's own controller but we want to have this in the explorer routing
+                # namespace because topic search is conceptually part of the explorer
+                controller = Controllers::SearchController.new(params)
+                controller.index(topic_id)
+              end
+
               controller = Controllers::ExplorerController.new(params)
 
               r.get String, Integer, 'recent' do |topic_id, partition_id|
@@ -262,16 +281,24 @@ module Karafka
             r.on 'topics' do
               controller = Controllers::TopicsController.new(params)
 
-              r.get String, 'config' do |topic_name|
-                controller.config(topic_name)
+              r.get String, 'config' do |topic_id|
+                controller.config(topic_id)
+              end
+
+              r.get String, 'replication' do |topic_id|
+                controller.replication(topic_id)
               end
 
-              r.get String, 'replication' do |topic_name|
-                controller.replication(topic_name)
+              r.get String, 'distribution' do |topic_id|
+                controller.distribution(topic_id)
               end
 
-              r.get String, 'distribution' do |topic_name|
-                controller.distribution(topic_name)
+              r.get String, 'offsets' do |topic_id|
+                controller.offsets(topic_id)
+              end
+
+              r.get String do |topic_id|
+                r.redirect root_path('topics', topic_id, 'config')
               end
 
               r.get do
@@ -308,6 +335,16 @@ module Karafka
               controller = Controllers::StatusController.new(params)
               controller.show
             end
+
+            r.get 'ux' do
+              controller = Controllers::UxController.new(params)
+              controller.show
+            end
+
+            r.get 'support' do
+              controller = Controllers::SupportController.new(params)
+              controller.show
+            end
           end
         end
       end

data/lib/karafka/web/pro/ui/controllers/dlq_controller.rb

@@ -49,7 +49,7 @@ module Karafka
             # @return [Boolean] is the given topic a DLQ topic
             def dlq?(topics_matches, name)
               topics_matches.any? do |candidate|
-                candidate.match?(name)
+                candidate.is_a?(Regexp) ? candidate.match?(name) : candidate == name
               end
             end
           end

data/lib/karafka/web/pro/ui/controllers/errors_controller.rb

@@ -38,6 +38,7 @@ module Karafka
 
             # @param partition_id [Integer] id of the partition of errors we are interested in
             def partition(partition_id)
+              @topic_id = errors_topic
               @partition_id = partition_id
               @watermark_offsets = Models::WatermarkOffsets.find(errors_topic, @partition_id)
               @partitions_count = Models::ClusterInfo.partitions_count(errors_topic)

data/lib/karafka/web/pro/ui/controllers/explorer_controller.rb

@@ -43,7 +43,7 @@ module Karafka
             # @note We cannot use offset references here because each of the partitions may have
             #   completely different values
             def topic(topic_id)
-              @visibility_filter = ::Karafka::Web.config.ui.visibility.filter
+              @visibility_filter = ::Karafka::Web.config.ui.policies.messages
 
               @topic_id = topic_id
               @partitions_count = Models::ClusterInfo.partitions_count(topic_id)
@@ -66,7 +66,7 @@ module Karafka
             # @param topic_id [String]
             # @param partition_id [Integer]
             def partition(topic_id, partition_id)
-              @visibility_filter = ::Karafka::Web.config.ui.visibility.filter
+              @visibility_filter = ::Karafka::Web.config.ui.policies.messages
               @topic_id = topic_id
               @partition_id = partition_id
               @watermark_offsets = Models::WatermarkOffsets.find(topic_id, partition_id)
@@ -94,23 +94,15 @@ module Karafka
             def show(topic_id, partition_id, offset, paginate: true)
               Lib::PatternsDetector.new.call
 
-              @visibility_filter = ::Karafka::Web.config.ui.visibility.filter
+              @visibility_filter = ::Karafka::Web.config.ui.policies.messages
               @topic_id = topic_id
               @partition_id = partition_id
               @offset = offset
               @message = Models::Message.find(@topic_id, @partition_id, @offset)
 
-              @safe_payload = Web::Ui::Lib::SafeRunner.new do
-                JSON.pretty_generate(@message.payload)
-              end
-
-              @safe_key = Web::Ui::Lib::SafeRunner.new do
-                @message.key
-              end
-
-              @safe_headers = Web::Ui::Lib::SafeRunner.new do
-                @message.headers
-              end
+              @safe_key = Web::Pro::Ui::Lib::SafeRunner.new { @message.key }.tap(&:call)
+              @safe_headers = Web::Pro::Ui::Lib::SafeRunner.new { @message.headers }.tap(&:call)
+              @safe_payload = Web::Pro::Ui::Lib::SafeRunner.new { @message.payload }.tap(&:call)
 
               # This may be off for certain views like recent view where we are interested only
               # in the most recent all the time. It does not make any sense to display pagination

data/lib/karafka/web/pro/ui/controllers/messages_controller.rb

@@ -28,6 +28,8 @@ module Karafka
             def republish(topic_id, partition_id, offset)
               message = Models::Message.find(topic_id, partition_id, offset)
 
+              deny! unless visibility_filter.republish?(message)
+
               delivery = ::Karafka::Web.producer.produce_sync(
                 topic: topic_id,
                 partition: partition_id,
@@ -50,8 +52,7 @@ module Karafka
             def download(topic_id, partition_id, offset)
               message = Models::Message.find(topic_id, partition_id, offset)
 
-              # Check if downloads are allowed
-              return deny unless visibility_filter.download?(message)
+              deny! unless visibility_filter.download?(message)
 
               file(
                 message.raw_payload,
@@ -73,7 +74,7 @@ module Karafka
               message = Models::Message.find(topic_id, partition_id, offset)
 
               # Check if exports are allowed
-              return deny unless visibility_filter.export?(message)
+              deny! unless visibility_filter.export?(message)
 
               file(
                 message.payload.to_json,
@@ -96,7 +97,7 @@ module Karafka
 
             # @return [Object] visibility filter. Either default or user-based
             def visibility_filter
-              ::Karafka::Web.config.ui.visibility.filter
+              ::Karafka::Web.config.ui.policies.messages
             end
           end
         end

data/lib/karafka/web/pro/ui/controllers/search_controller.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Controllers
+          # Handles the search requests
+          # We present this as a part of explorer scope but we use a separate controller not to
+          # mix data exploring with searching.
+          class SearchController < Web::Ui::Controllers::ClusterController
+            # Runs the search if search parameters are provided
+            # If no parameters provided, displays the search modal and info to provide search data
+            # If invalid search parameters provided, modal contains errors
+            #
+            # @param topic_id [String] topic we're interested in
+            # @note In theory search can be used to detect pieces of information within messages.
+            #   Since we allow for custom search strategies, this is not an issue because users
+            #   that need to provide only granular search can do so.
+            def index(topic_id)
+              @topic_id = topic_id
+              @partitions_count = Models::ClusterInfo.partitions_count(topic_id)
+              # Select only matchers that should be available in the context of the current topic
+              @matchers = Web.config.ui.search.matchers.select { |match| match.active?(@topic_id) }
+              @search_criteria = !@params.current_search.empty?
+              @current_search = Lib::Search::Normalizer.call(@params.current_search)
+              # Needed when rendering found messages rows. We should always filter the messages
+              # details with the visibility filter
+              @visibility_filter = ::Karafka::Web.config.ui.policies.messages
+              @limits = ::Karafka::Web.config.ui.search.limits.sort
+
+              # If there is search form filled, we validate it to make sure there are no errors
+              @errors = if @search_criteria
+                          Lib::Search::Contracts::Form.new.call(@current_search).errors
+                        else
+                          {}
+                        end
+
+              # If all good we run the search
+              if @search_criteria && @errors.empty?
+                found, @search_details = Lib::Search::Runner.new(
+                  @topic_id,
+                  @partitions_count,
+                  @current_search
+                ).call
+
+                @messages, last_page = Paginators::Arrays.call(
+                  found,
+                  @params.current_page
+                )
+
+                paginate(@params.current_page, !last_page)
+              end
+
+              render
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/controllers/support_controller.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Controllers
+          # Support details - same as in OSS
+          class SupportController < Web::Ui::Controllers::SupportController
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/controllers/topics_controller.rb

@@ -33,6 +33,8 @@ module Karafka
               count
               share
               diff
+              low
+              high
             ].freeze
 
             # Lists available topics in the cluster
@@ -91,6 +93,35 @@ module Karafka
 
               render
             end
+
+            # Displays high and low offsets for given topic
+            #
+            # @param topic_name [String] topic we're interested in
+            def offsets(topic_name)
+              @topic = Models::Topic.find(topic_name)
+
+              @active_partitions, _materialized_page, @limited = Paginators::Partitions.call(
+                @topic.partition_count, @params.current_page
+              )
+
+              offsets = @active_partitions.map do |partition_id|
+                part_offsets = Admin.read_watermark_offsets(topic_name, partition_id)
+
+                {
+                  partition_id: partition_id,
+                  low: part_offsets.first,
+                  high: part_offsets.last,
+                  diff: part_offsets.last - part_offsets.first
+                }
+              end
+
+              @offsets = refine(offsets)
+
+              next_page = @active_partitions.last < @topic.partition_count - 1
+              paginate(@params.current_page, next_page)
+
+              render
+            end
           end
         end
       end

data/lib/karafka/web/pro/ui/controllers/ux_controller.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Controllers
+          # Ux details - same as in OSS
+          class UxController < Web::Ui::Controllers::UxController
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/lib/policies/config.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Lib
+          module Policies
+            # Extra configuration for pro UI
+            class Config
+              extend ::Karafka::Core::Configurable
+
+              # Policies controller related to messages operations and visibility
+              setting :messages, default: Policies::Messages.new
+
+              # Policies controller related to all requests. It is a general one that is not
+              # granular but can be used to block completely certain pieces of the UI from
+              # accessing like explorer or any other as operates on the raw env level
+              setting :requests, default: Policies::Requests.new
+
+              configure
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/lib/policies/contracts/config.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Lib
+          # Policies related contracts
+          module Policies
+            # Namespace with policies related contracts
+            module Contracts
+              # Makes sure, all the expected UI policies config is defined as it should be
+              class Config < ::Karafka::Contracts::Base
+                configure do |config|
+                  config.error_messages = YAML.safe_load(
+                    File.read(
+                      File.join(Karafka::Web.gem_root, 'config', 'locales', 'pro_errors.yml')
+                    )
+                  ).fetch('en').fetch('validations').fetch('config')
+                end
+
+                nested(:ui) do
+                  nested(:policies) do
+                    required(:messages) { |val| !val.nil? }
+                    required(:requests) { |val| !val.nil? }
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/lib/policies/messages.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Lib
+          module Policies
+            # Allows for a granular control over what parts of messages are being displayed and
+            # operated on.
+            # There are scenarios where payload or other parts of messages should not be presented
+            # because they may contain sensitive data. This API allows to manage that on a per
+            # message basis.
+            class Messages
+              # @param _message [::Karafka::Messages::Message]
+              # @return [Boolean] should message key be visible
+              def key?(_message)
+                true
+              end
+
+              # @param _message [::Karafka::Messages::Message]
+              # @return [Boolean] should message headers be visible
+              def headers?(_message)
+                true
+              end
+
+              # @param message [::Karafka::Messages::Message]
+              # @return [Boolean] should message payload be visible
+              def payload?(message)
+                !message.headers.key?('encryption')
+              end
+
+              # Should it be allowed to download this message raw payload
+              #
+              # @param message [::Karafka::Messages::Message]
+              # @return [Boolean] true if downloads allowed
+              def download?(message)
+                payload?(message)
+              end
+
+              # Should it be allowed to download the deserialized and sanitized payload as JSON
+              #
+              # @param message [::Karafka::Messages::Message]
+              # @return [Boolean] true if exports allowed
+              def export?(message)
+                payload?(message)
+              end
+
+              # Should we allow to republish given message
+              #
+              # @param _message [::Karafka::Messages::Message]
+              # @return [Boolean] true if we should allow republishing
+              # @note This is a simple API that does not provide granular republishing support.
+              #   You can decide whether to allow for republishing but you cannot say "allow only
+              #   to X", etc.
+              def republish?(_message)
+                true
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/lib/policies/requests.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Lib
+          module Policies
+            # Per request policies setup class. Allows for writing low level policies that can
+            # be used to block certain Web UI functionalities on the request level.
+            class Requests
+              # @param _env [Hash] rack env object that we can use to get request details
+              # @return [Boolean] should this request be allowed or not
+              # @note By default we do not limit anything in the Web UI, however particular
+              #   granular policies may limit things on their own.
+              def allow?(_env)
+                true
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/lib/policies.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Lib
+          # All code needed to support the search functionality
+          module Policies
+            class << self
+              # Validates that the UI policies config is correct
+              #
+              # @param config [Karafka::Core::Configurable::Node] web config
+              def post_setup(config)
+                Policies::Contracts::Config.new.validate!(config.to_h)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end