karafka-web 0.9.2 → 0.10.0.beta1

data/lib/karafka/web/pro/ui/lib/safe_runner.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Lib
+          # Class used to execute code that can fail but we do not want to fail the whole
+          # operation. The primary use-case is for displaying deserialized data. We always need to
+          # assume, that part of the data can be corrupted and it should not crash the whole UI.
+          #
+          # It caches the result and does not run the code twice (only once). Additionally, it
+          # measures the CPU usage and total time during the execution of the code block.
+          class SafeRunner
+            include Karafka::Core::Helpers::Time
+
+            attr_reader :error, :result, :cpu_time, :total_time, :allocations
+
+            # @param block [Proc] code we want to safe-guard
+            def initialize(&block)
+              @code = block
+              @executed = false
+              @success = false
+              @error = nil
+              @result = nil
+              @cpu_time = 0
+              @total_time = 0
+              @allocations = false
+            end
+
+            # @return [Boolean] was the code execution successful or not
+            def success?
+              return @success if executed?
+
+              call
+
+              @success
+            end
+
+            # @return [Boolean] was the code execution failed or not
+            def failure?
+              !success?
+            end
+
+            # Runs the execution and returns block result
+            def call
+              return @result if executed?
+
+              @executed = true
+
+              if objspace?
+                ObjectSpace.trace_object_allocations_start
+                before = ObjectSpace.each_object.count
+              end
+
+              # We measure time as close to the process as possible so it is not impacted by the
+              # objects allocations count (if applicable)
+              start_time = monotonic_now
+              start_cpu = ::Process.times
+              @result = @code.call
+              @success = true
+
+              @result
+            rescue StandardError => e
+              @error = e
+              @success = false
+            ensure
+              end_time = monotonic_now
+              end_cpu = ::Process.times
+
+              @cpu_time = (
+                (end_cpu.utime - start_cpu.utime) + (end_cpu.stime - start_cpu.stime)
+              ) * 1_000
+              @total_time = (end_time - start_time)
+
+              if objspace?
+                @allocations = ObjectSpace.each_object.count - before
+                ObjectSpace.trace_object_allocations_stop
+              end
+            end
+
+            # @return [Boolean] was the code executed already or not yet
+            def executed?
+              @executed
+            end
+
+            private
+
+            # @return [Boolean] true if tracing is available
+            def objspace?
+              ObjectSpace.respond_to?(:trace_object_allocations_start)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/lib/search/config.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Lib
+          module Search
+            # Extra configuration for pro UI
+            class Config
+              extend ::Karafka::Core::Configurable
+
+              # Matches for messages
+              setting :matchers, default: [
+                Matchers::RawPayloadIncludes,
+                Matchers::RawKeyIncludes,
+                Matchers::RawHeaderIncludes
+              ]
+
+              # How long should we at most search before stopping (in ms)
+              # This prevents us from having a search that would basically hang the browser and
+              # never finish
+              setting :timeout, default: 30_000
+
+              # Search limits as the search runs in Puma
+              # Too big value will make the scan really slow
+              # We do not put a max limit here because those values are used in a select in
+              # the Web UI.
+              setting :limits, default: [
+                1_000,
+                10_000,
+                100_000
+              ]
+
+              configure
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/lib/search/contracts/config.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Lib
+          # Search related contracts
+          module Search
+            # Namespace with search related contracts
+            module Contracts
+              # Makes sure, all the expected UI search config is defined as it should be
+              class Config < ::Karafka::Contracts::Base
+                configure do |config|
+                  config.error_messages = YAML.safe_load(
+                    File.read(
+                      File.join(Karafka::Web.gem_root, 'config', 'locales', 'pro_errors.yml')
+                    )
+                  ).fetch('en').fetch('validations').fetch('config')
+                end
+
+                nested(:ui) do
+                  nested(:search) do
+                    required(:matchers) { |val| val.is_a?(Array) && !val.empty? }
+
+                    required(:timeout) { |val| val.is_a?(Integer) && val.positive? }
+
+                    # Users can define their own search limits and we just make sure they do not
+                    # do something weird like negative numbers
+                    required(:limits) do |val|
+                      next false unless val.is_a?(Array)
+                      next false if val.empty?
+                      next false unless val.all? { |count| count.is_a?(Integer) }
+                      next false unless val.all?(&:positive?)
+
+                      true
+                    end
+                  end
+                end
+
+                # Ensure all matchers respond to name and are callable
+                virtual do |data, errors|
+                  next unless errors.empty?
+
+                  detected_errors = []
+
+                  data.dig(:ui, :search, :matchers).each do |matcher|
+                    next if matcher.respond_to?(:name) && matcher.public_method_defined?(:call)
+
+                    detected_errors << [%i[ui search matchers], :must_have_name_and_call]
+                  end
+
+                  detected_errors
+                end
+
+                # Make sure that all matchers names are unique
+                virtual do |data, errors|
+                  next unless errors.empty?
+
+                  names = data.dig(:ui, :search, :matchers).map(&:name)
+
+                  next if names.uniq.size == names.size
+
+                  [[%i[ui search matchers], :must_have_unique_names]]
+                end
+
+                # Make sure that all matchers names are strings and not empty
+                virtual do |data, errors|
+                  next unless errors.empty?
+
+                  detected_errors = []
+
+                  data.dig(:ui, :search, :matchers).map(&:name).each do |name|
+                    next if name.is_a?(String) && !name.empty?
+
+                    detected_errors << [%i[ui search matchers], :name_must_be_valid]
+
+                    break
+                  end
+
+                  detected_errors
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/lib/search/contracts/form.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Lib
+          module Search
+            module Contracts
+              # Validates the search input
+              # This is not a complex validator with error handling, etc
+              # Since we use HTML5 validations that are good enough, here we check all the
+              # consistency of the data but without providing sophisticated user error reporting as
+              # it is the final checking that should not happen often.
+              #
+              # @note This does not validate the raw input from the HTML but one that was slightly
+              #   normalized to simplify the flow.
+              class Form < Web::Contracts::Base
+                configure do |config|
+                  config.error_messages = YAML.safe_load(
+                    File.read(
+                      File.join(Karafka::Web.gem_root, 'config', 'locales', 'pro_errors.yml')
+                    )
+                  ).fetch('en').fetch('validations').fetch('search_form')
+                end
+
+                # Minimum timestamp value when timestamps are used
+                # 2001-09-09. This value selected because it is older than Kafka and it looks nice
+                MIN_TIMESTAMP = 1_000_000_000
+
+                private_constant :MIN_TIMESTAMP
+
+                # What are we looking for
+                required(:phrase) { |val| val.is_a?(String) && !val.empty? }
+
+                # How many messages in total should we scan. In case of many partitions, this is
+                # distributed evenly across them. So 100 000 in 5 partitions, means scanning
+                # 20 000 per partition.
+                required(:limit) do |val|
+                  Web.config.ui.search.limits.include?(val)
+                end
+
+                # What matcher should we use when matching phrase and message
+                # It allows only matchers that we have declared in search
+                required(:matcher) do |val|
+                  Web.config.ui.search.matchers.map(&:name).include?(val)
+                end
+
+                # Where should we start looking in the data
+                # - latest means we move back the needed per partition number of messages and we look
+                # - offset means we start from the same offset on all the partitions
+                # - timestamp means we start from a given time moment on all the partitions
+                required(:offset_type) { |val| %w[latest offset timestamp].include?(val) }
+
+                # Offset from which to start. We can always require it even if we do not start from
+                # a certain offset because we work on a normalized data
+                required(:offset) { |val| val.is_a?(Integer) && val >= 0 }
+
+                # Similar as above but for timestamp.
+                required(:timestamp) do |val|
+                  next false unless val.is_a?(Integer)
+                  next false if val.negative?
+                  next false if val > ((Time.now.to_f + 60) * 1_000).to_i
+
+                  true
+                end
+
+                # We do not check here that partitions actually exist because when searching we just
+                # reject non-existing partitions
+                required(:partitions) do |val|
+                  next false unless val.is_a?(Array)
+                  next false if val.empty?
+
+                  val.all? { |ar_val| ar_val.is_a?(String) }
+                end
+
+                # Special validation for timestamp to make sure it is not older than 2010
+                # Since Kafka is not that old, timestamps should never be less than that.
+                # While all the others will use standard "is invalid" because they also have a
+                # frontend HTML5 validation. This one is specific because we want to allow setting
+                # 0 as long as we don't select the timestamp as a value. Then we want to make sure
+                # that user provides the message timestamp which is in ms
+                virtual do |data, errors|
+                  next unless errors.empty?
+                  # Validate only if we decide to go with timestamp. Otherwise this value is
+                  # irrelevant
+                  next unless data[:offset_type] == 'timestamp'
+
+                  next if data[:timestamp] >= MIN_TIMESTAMP
+
+                  [[%i[timestamp], :key_must_be_large_enough]]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/lib/search/matchers/base.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Lib
+          module Search
+            # Namespace for search matchers
+            # Matcher is what "searches" a message or its parts for a given phrase
+            module Matchers
+              # Base class for all the search matchers
+              # Each matcher needs to have a class `#name` method and needs to respond to `#call`
+              class Base
+                # @param phrase [String] string phrase for search
+                # @param message [Karafka::Messages::Message] message in which we search
+                # @return [Boolean] true if found, otherwise false
+                def call(phrase, message)
+                  raise NotImplementedError, 'Implement in a subclass'
+                end
+
+                class << self
+                  # @return [String] name of the matcher based on the class name
+                  def name
+                    # Insert a space before each uppercase letter, except the first one
+                    spaced_string = to_s.split('::').last.gsub(/(?<!^)([A-Z])/, ' \1')
+
+                    # Convert the first letter to uppercase and the rest to lowercase
+                    spaced_string[0].upcase + spaced_string[1..].downcase
+                  end
+
+                  # Allows to disable/enable certain matches that may be specific to certain types
+                  # of data per topic. By default matchers are always active
+                  #
+                  # @param _topic_name [String] name of the topic in the explorer
+                  # @return [Boolean] should this matcher be active in the given topic
+                  def active?(_topic_name)
+                    true
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/lib/search/matchers/raw_header_includes.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Lib
+          module Search
+            module Matchers
+              # Matcher that searches in the raw headers. If any header key or value matches
+              # the phrase, it is true. Otherwise false.
+              #
+              # @note It is case sensitive
+              # @note Ignores encoding issues
+              class RawHeaderIncludes < Base
+                # @param message [Karafka::Messages::Message]
+                # @param phrase [String]
+                # @return [Boolean] does message raw headers contain the phrase
+                def call(message, phrase)
+                  message.raw_headers.each do |raw_header_key, raw_header_value|
+                    return true if safe_include?(raw_header_key, phrase)
+                    return true if safe_include?(raw_header_value, phrase)
+                  end
+
+                  false
+                end
+
+                private
+
+                # Checks the inclusion ignoring encoding issues
+                # @param string [String] String in which we look
+                # @param phrase [String] string for which we look
+                # @return [Boolean] true if found, otherwise false
+                def safe_include?(string, phrase)
+                  string.include?(phrase)
+                rescue Encoding::CompatibilityError
+                  false
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/lib/search/matchers/raw_key_includes.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Lib
+          module Search
+            module Matchers
+              # Checks if the key contains requested phrase
+              #
+              # @note It is case sensitive
+              # @note Ignores encoding issues
+              class RawKeyIncludes < Base
+                # @param message [Karafka::Messages::Message]
+                # @param phrase [String]
+                # @return [Boolean] does message raw key contain the phrase
+                def call(message, phrase)
+                  message.raw_key.to_s.include?(phrase)
+                rescue Encoding::CompatibilityError
+                  false
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/lib/search/matchers/raw_payload_includes.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Lib
+          module Search
+            module Matchers
+              # Checks for phrase existence in the raw payload
+              #
+              # @note It is case sensitive
+              # @note Ignores encoding issues
+              class RawPayloadIncludes < Base
+                # @param message [Karafka::Messages::Message]
+                # @param phrase [String]
+                # @return [Boolean] does message raw payload contain the phrase
+                def call(message, phrase)
+                  # raw payload can be nil for tombstone events
+                  return false unless message.raw_payload
+
+                  message.raw_payload.include?(phrase)
+                # String matching on compressed data may fail
+                rescue Encoding::CompatibilityError
+                  false
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/lib/search/normalizer.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Lib
+          module Search
+            # Takes basic search parameters and normalizes the data a bit
+            # Since we have fairly simple search input argument types, we can cast them easily
+            # into format that we can accept further down the pipeline.
+            # This module provides this normalization, so we do not have to worry about weird
+            # edge cases.
+            module Normalizer
+              class << self
+                # @param search_query [Hash] hash with expected data
+                # @return [Hash] normalized search query hash
+                def call(search_query)
+                  {
+                    phrase: search_query['phrase'].to_s,
+                    limit: search_query['limit'].to_i,
+                    matcher: search_query['matcher'].to_s,
+                    partitions: Array(search_query['partitions']).flatten.compact.uniq,
+                    offset_type: search_query['offset_type'].to_s,
+                    timestamp: search_query['timestamp'].to_i,
+                    offset: search_query['offset'].to_i
+                  }
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end