karafka-web 0.9.1 → 0.10.0.rc1

data/lib/karafka/web/pro/ui/controllers/explorer_controller.rb

@@ -43,7 +43,7 @@ module Karafka
             # @note We cannot use offset references here because each of the partitions may have
             #   completely different values
             def topic(topic_id)
-              @visibility_filter = ::Karafka::Web.config.ui.visibility.filter
+              @visibility_filter = ::Karafka::Web.config.ui.policies.messages
 
               @topic_id = topic_id
               @partitions_count = Models::ClusterInfo.partitions_count(topic_id)
@@ -66,7 +66,7 @@ module Karafka
             # @param topic_id [String]
             # @param partition_id [Integer]
             def partition(topic_id, partition_id)
-              @visibility_filter = ::Karafka::Web.config.ui.visibility.filter
+              @visibility_filter = ::Karafka::Web.config.ui.policies.messages
               @topic_id = topic_id
               @partition_id = partition_id
               @watermark_offsets = Models::WatermarkOffsets.find(topic_id, partition_id)
@@ -94,23 +94,15 @@ module Karafka
             def show(topic_id, partition_id, offset, paginate: true)
               Lib::PatternsDetector.new.call
 
-              @visibility_filter = ::Karafka::Web.config.ui.visibility.filter
+              @visibility_filter = ::Karafka::Web.config.ui.policies.messages
               @topic_id = topic_id
               @partition_id = partition_id
               @offset = offset
               @message = Models::Message.find(@topic_id, @partition_id, @offset)
 
-              @safe_payload = Web::Ui::Lib::SafeRunner.new do
-                JSON.pretty_generate(@message.payload)
-              end
-
-              @safe_key = Web::Ui::Lib::SafeRunner.new do
-                @message.key
-              end
-
-              @safe_headers = Web::Ui::Lib::SafeRunner.new do
-                @message.headers
-              end
+              @safe_key = Web::Pro::Ui::Lib::SafeRunner.new { @message.key }.tap(&:call)
+              @safe_headers = Web::Pro::Ui::Lib::SafeRunner.new { @message.headers }.tap(&:call)
+              @safe_payload = Web::Pro::Ui::Lib::SafeRunner.new { @message.payload }.tap(&:call)
 
               # This may be off for certain views like recent view where we are interested only
               # in the most recent all the time. It does not make any sense to display pagination

data/lib/karafka/web/pro/ui/controllers/messages_controller.rb

@@ -28,6 +28,8 @@ module Karafka
             def republish(topic_id, partition_id, offset)
               message = Models::Message.find(topic_id, partition_id, offset)
 
+              deny! unless visibility_filter.republish?(message)
+
               delivery = ::Karafka::Web.producer.produce_sync(
                 topic: topic_id,
                 partition: partition_id,
@@ -50,8 +52,7 @@ module Karafka
             def download(topic_id, partition_id, offset)
               message = Models::Message.find(topic_id, partition_id, offset)
 
-              # Check if downloads are allowed
-              return deny unless visibility_filter.download?(message)
+              deny! unless visibility_filter.download?(message)
 
               file(
                 message.raw_payload,
@@ -73,7 +74,7 @@ module Karafka
               message = Models::Message.find(topic_id, partition_id, offset)
 
               # Check if exports are allowed
-              return deny unless visibility_filter.export?(message)
+              deny! unless visibility_filter.export?(message)
 
               file(
                 message.payload.to_json,
@@ -96,7 +97,7 @@ module Karafka
 
             # @return [Object] visibility filter. Either default or user-based
             def visibility_filter
-              ::Karafka::Web.config.ui.visibility.filter
+              ::Karafka::Web.config.ui.policies.messages
             end
           end
         end

data/lib/karafka/web/pro/ui/controllers/search_controller.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Controllers
+          # Handles the search requests
+          # We present this as a part of explorer scope but we use a separate controller not to
+          # mix data exploring with searching.
+          class SearchController < Web::Ui::Controllers::ClusterController
+            # Runs the search if search parameters are provided
+            # If no parameters provided, displays the search modal and info to provide search data
+            # If invalid search parameters provided, modal contains errors
+            #
+            # @param topic_id [String] topic we're interested in
+            # @note In theory search can be used to detect pieces of information within messages.
+            #   Since we allow for custom search strategies, this is not an issue because users
+            #   that need to provide only granular search can do so.
+            def index(topic_id)
+              @topic_id = topic_id
+              @partitions_count = Models::ClusterInfo.partitions_count(topic_id)
+              # Select only matchers that should be available in the context of the current topic
+              @matchers = Web.config.ui.search.matchers.select { |match| match.active?(@topic_id) }
+              @search_criteria = !@params.current_search.empty?
+              @current_search = Lib::Search::Normalizer.call(@params.current_search)
+              # Needed when rendering found messages rows. We should always filter the messages
+              # details with the visibility filter
+              @visibility_filter = ::Karafka::Web.config.ui.policies.messages
+              @limits = ::Karafka::Web.config.ui.search.limits.sort
+
+              # If there is search form filled, we validate it to make sure there are no errors
+              @errors = if @search_criteria
+                          Lib::Search::Contracts::Form.new.call(@current_search).errors
+                        else
+                          {}
+                        end
+
+              # If all good we run the search
+              if @search_criteria && @errors.empty?
+                found, @search_details = Lib::Search::Runner.new(
+                  @topic_id,
+                  @partitions_count,
+                  @current_search
+                ).call
+
+                @messages, last_page = Paginators::Arrays.call(
+                  found,
+                  @params.current_page
+                )
+
+                paginate(@params.current_page, !last_page)
+              end
+
+              render
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/controllers/support_controller.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Controllers
+          # Support details - same as in OSS
+          class SupportController < Web::Ui::Controllers::SupportController
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/controllers/topics_controller.rb

@@ -33,6 +33,8 @@ module Karafka
               count
               share
               diff
+              low
+              high
             ].freeze
 
             # Lists available topics in the cluster
@@ -91,6 +93,35 @@ module Karafka
 
               render
             end
+
+            # Displays high and low offsets for given topic
+            #
+            # @param topic_name [String] topic we're interested in
+            def offsets(topic_name)
+              @topic = Models::Topic.find(topic_name)
+
+              @active_partitions, _materialized_page, @limited = Paginators::Partitions.call(
+                @topic.partition_count, @params.current_page
+              )
+
+              offsets = @active_partitions.map do |partition_id|
+                part_offsets = Admin.read_watermark_offsets(topic_name, partition_id)
+
+                {
+                  partition_id: partition_id,
+                  low: part_offsets.first,
+                  high: part_offsets.last,
+                  diff: part_offsets.last - part_offsets.first
+                }
+              end
+
+              @offsets = refine(offsets)
+
+              next_page = @active_partitions.last < @topic.partition_count - 1
+              paginate(@params.current_page, next_page)
+
+              render
+            end
           end
         end
       end

data/lib/karafka/web/pro/ui/controllers/ux_controller.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Controllers
+          # Ux details - same as in OSS
+          class UxController < Web::Ui::Controllers::UxController
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/lib/policies/config.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Lib
+          module Policies
+            # Extra configuration for pro UI
+            class Config
+              extend ::Karafka::Core::Configurable
+
+              # Policies controller related to messages operations and visibility
+              setting :messages, default: Policies::Messages.new
+
+              # Policies controller related to all requests. It is a general one that is not
+              # granular but can be used to block completely certain pieces of the UI from
+              # accessing like explorer or any other as operates on the raw env level
+              setting :requests, default: Policies::Requests.new
+
+              configure
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/lib/policies/contracts/config.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Lib
+          # Policies related contracts
+          module Policies
+            # Namespace with policies related contracts
+            module Contracts
+              # Makes sure, all the expected UI policies config is defined as it should be
+              class Config < ::Karafka::Contracts::Base
+                configure do |config|
+                  config.error_messages = YAML.safe_load(
+                    File.read(
+                      File.join(Karafka::Web.gem_root, 'config', 'locales', 'pro_errors.yml')
+                    )
+                  ).fetch('en').fetch('validations').fetch('config')
+                end
+
+                nested(:ui) do
+                  nested(:policies) do
+                    required(:messages) { |val| !val.nil? }
+                    required(:requests) { |val| !val.nil? }
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/lib/policies/messages.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Lib
+          module Policies
+            # Allows for a granular control over what parts of messages are being displayed and
+            # operated on.
+            # There are scenarios where payload or other parts of messages should not be presented
+            # because they may contain sensitive data. This API allows to manage that on a per
+            # message basis.
+            class Messages
+              # @param _message [::Karafka::Messages::Message]
+              # @return [Boolean] should message key be visible
+              def key?(_message)
+                true
+              end
+
+              # @param _message [::Karafka::Messages::Message]
+              # @return [Boolean] should message headers be visible
+              def headers?(_message)
+                true
+              end
+
+              # @param message [::Karafka::Messages::Message]
+              # @return [Boolean] should message payload be visible
+              def payload?(message)
+                !message.headers.key?('encryption')
+              end
+
+              # Should it be allowed to download this message raw payload
+              #
+              # @param message [::Karafka::Messages::Message]
+              # @return [Boolean] true if downloads allowed
+              def download?(message)
+                payload?(message)
+              end
+
+              # Should it be allowed to download the deserialized and sanitized payload as JSON
+              #
+              # @param message [::Karafka::Messages::Message]
+              # @return [Boolean] true if exports allowed
+              def export?(message)
+                payload?(message)
+              end
+
+              # Should we allow to republish given message
+              #
+              # @param _message [::Karafka::Messages::Message]
+              # @return [Boolean] true if we should allow republishing
+              # @note This is a simple API that does not provide granular republishing support.
+              #   You can decide whether to allow for republishing but you cannot say "allow only
+              #   to X", etc.
+              def republish?(_message)
+                true
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/lib/policies/requests.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Lib
+          module Policies
+            # Per request policies setup class. Allows for writing low level policies that can
+            # be used to block certain Web UI functionalities on the request level.
+            class Requests
+              # @param _env [Hash] rack env object that we can use to get request details
+              # @return [Boolean] should this request be allowed or not
+              # @note By default we do not limit anything in the Web UI, however particular
+              #   granular policies may limit things on their own.
+              def allow?(_env)
+                true
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/lib/policies.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Lib
+          # All code needed to support the search functionality
+          module Policies
+            class << self
+              # Validates that the UI policies config is correct
+              #
+              # @param config [Karafka::Core::Configurable::Node] web config
+              def post_setup(config)
+                Policies::Contracts::Config.new.validate!(config.to_h)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/lib/safe_runner.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Lib
+          # Class used to execute code that can fail but we do not want to fail the whole
+          # operation. The primary use-case is for displaying deserialized data. We always need to
+          # assume, that part of the data can be corrupted and it should not crash the whole UI.
+          #
+          # It caches the result and does not run the code twice (only once). Additionally, it
+          # measures the CPU usage and total time during the execution of the code block.
+          class SafeRunner
+            include Karafka::Core::Helpers::Time
+
+            attr_reader :error, :result, :cpu_time, :total_time, :allocations
+
+            # @param block [Proc] code we want to safe-guard
+            def initialize(&block)
+              @code = block
+              @executed = false
+              @success = false
+              @error = nil
+              @result = nil
+              @cpu_time = 0
+              @total_time = 0
+              @allocations = false
+            end
+
+            # @return [Boolean] was the code execution successful or not
+            def success?
+              return @success if executed?
+
+              call
+
+              @success
+            end
+
+            # @return [Boolean] was the code execution failed or not
+            def failure?
+              !success?
+            end
+
+            # Runs the execution and returns block result
+            def call
+              return @result if executed?
+
+              @executed = true
+
+              if objspace?
+                ObjectSpace.trace_object_allocations_start
+                before = ObjectSpace.each_object.count
+              end
+
+              # We measure time as close to the process as possible so it is not impacted by the
+              # objects allocations count (if applicable)
+              start_time = monotonic_now
+              start_cpu = ::Process.times
+              @result = @code.call
+              @success = true
+
+              @result
+            rescue StandardError => e
+              @error = e
+              @success = false
+            ensure
+              end_time = monotonic_now
+              end_cpu = ::Process.times
+
+              @cpu_time = (
+                (end_cpu.utime - start_cpu.utime) + (end_cpu.stime - start_cpu.stime)
+              ) * 1_000
+              @total_time = (end_time - start_time)
+
+              if objspace?
+                @allocations = ObjectSpace.each_object.count - before
+                ObjectSpace.trace_object_allocations_stop
+              end
+            end
+
+            # @return [Boolean] was the code executed already or not yet
+            def executed?
+              @executed
+            end
+
+            private
+
+            # @return [Boolean] true if tracing is available
+            def objspace?
+              ObjectSpace.respond_to?(:trace_object_allocations_start)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/lib/search/config.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Lib
+          module Search
+            # Extra configuration for pro UI
+            class Config
+              extend ::Karafka::Core::Configurable
+
+              # Matches for messages
+              setting :matchers, default: [
+                Matchers::RawPayloadIncludes,
+                Matchers::RawKeyIncludes,
+                Matchers::RawHeaderIncludes
+              ]
+
+              # How long should we at most search before stopping (in ms)
+              # This prevents us from having a search that would basically hang the browser and
+              # never finish
+              setting :timeout, default: 30_000
+
+              # Search limits as the search runs in Puma
+              # Too big value will make the scan really slow
+              # We do not put a max limit here because those values are used in a select in
+              # the Web UI.
+              setting :limits, default: [
+                1_000,
+                10_000,
+                100_000
+              ]
+
+              configure
+            end
+          end
+        end
+      end
+    end
+  end
+end