karafka 1.2.2 → 1.4.0.rc1

data/lib/karafka/contracts/consumer_group.rb

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Contracts
+    # Contract for single full route (consumer group + topics) validation.
+    class ConsumerGroup < Dry::Validation::Contract
+      config.messages.load_paths << File.join(Karafka.gem_root, 'config', 'errors.yml')
+
+      # Valid uri schemas of Kafka broker url
+      # The ||= is due to the behavior of require_all that resolves dependencies
+      # but sometimes loads things twice
+      URI_SCHEMES ||= %w[kafka kafka+ssl plaintext ssl].freeze
+
+      # Available sasl scram mechanism of authentication (plus nil)
+      SASL_SCRAM_MECHANISMS ||= %w[sha256 sha512].freeze
+
+      # Internal contract for sub-validating topics schema
+      TOPIC_CONTRACT = ConsumerGroupTopic.new.freeze
+
+      private_constant :TOPIC_CONTRACT
+
+      params do
+        required(:id).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)
+        required(:topics).value(:array, :filled?)
+        required(:seed_brokers).value(:array, :filled?)
+        required(:session_timeout).filled { int? | float? }
+        required(:pause_timeout).maybe(%i[integer float]) { filled? > gteq?(0) }
+        required(:pause_max_timeout).maybe(%i[integer float]) { filled? > gteq?(0) }
+        required(:pause_exponential_backoff).filled(:bool?)
+        required(:offset_commit_interval) { int? | float? }
+        required(:offset_commit_threshold).filled(:int?)
+        required(:offset_retention_time).maybe(:integer)
+        required(:heartbeat_interval).filled { (int? | float?) & gteq?(0) }
+        required(:fetcher_max_queue_size).filled(:int?, gt?: 0)
+        required(:connect_timeout).filled { (int? | float?) & gt?(0) }
+        required(:reconnect_timeout).filled { (int? | float?) & gteq?(0) }
+        required(:socket_timeout).filled { (int? | float?) & gt?(0) }
+        required(:min_bytes).filled(:int?, gt?: 0)
+        required(:max_bytes).filled(:int?, gt?: 0)
+        required(:max_wait_time).filled { (int? | float?) & gteq?(0) }
+        required(:batch_fetching).filled(:bool?)
+
+        %i[
+          ssl_ca_cert
+          ssl_ca_cert_file_path
+          ssl_client_cert
+          ssl_client_cert_key
+          ssl_client_cert_chain
+          ssl_client_cert_key_password
+          sasl_gssapi_principal
+          sasl_gssapi_keytab
+          sasl_plain_authzid
+          sasl_plain_username
+          sasl_plain_password
+          sasl_scram_username
+          sasl_scram_password
+        ].each do |encryption_attribute|
+          optional(encryption_attribute).maybe(:str?)
+        end
+
+        optional(:ssl_verify_hostname).maybe(:bool?)
+        optional(:ssl_ca_certs_from_system).maybe(:bool?)
+        optional(:sasl_over_ssl).maybe(:bool?)
+        optional(:sasl_oauth_token_provider).value(:any)
+
+        # It's not with other encryptions as it has some more rules
+        optional(:sasl_scram_mechanism)
+          .maybe(:str?, included_in?: SASL_SCRAM_MECHANISMS)
+      end
+
+      # Uri rule to check if uri is in a Karafka acceptable format
+      rule(:seed_brokers) do
+        if value&.is_a?(Array) && !value.all?(&method(:kafka_uri?))
+          key.failure(:invalid_broker_schema)
+        end
+      end
+
+      rule(:topics) do
+        if value&.is_a?(Array)
+          names = value.map { |topic| topic[:name] }
+
+          key.failure(:topics_names_not_unique) if names.size != names.uniq.size
+        end
+      end
+
+      rule(:topics) do
+        if value&.is_a?(Array)
+          value.each_with_index do |topic, index|
+            TOPIC_CONTRACT.call(topic).errors.each do |error|
+              key([:topics, index, error.path[0]]).failure(error.text)
+            end
+          end
+        end
+      end
+
+      rule(:ssl_client_cert, :ssl_client_cert_key) do
+        if values[:ssl_client_cert] && !values[:ssl_client_cert_key]
+          key(:ssl_client_cert_key).failure(:ssl_client_cert_with_ssl_client_cert_key)
+        end
+      end
+
+      rule(:ssl_client_cert, :ssl_client_cert_key) do
+        if values[:ssl_client_cert_key] && !values[:ssl_client_cert]
+          key(:ssl_client_cert).failure(:ssl_client_cert_key_with_ssl_client_cert)
+        end
+      end
+
+      rule(:ssl_client_cert, :ssl_client_cert_chain) do
+        if values[:ssl_client_cert_chain] && !values[:ssl_client_cert]
+          key(:ssl_client_cert).failure(:ssl_client_cert_chain_with_ssl_client_cert)
+        end
+      end
+
+      rule(:ssl_client_cert_chain, :ssl_client_cert_key) do
+        if values[:ssl_client_cert_chain] && !values[:ssl_client_cert]
+          key(:ssl_client_cert).failure(:ssl_client_cert_chain_with_ssl_client_cert_key)
+        end
+      end
+
+      rule(:ssl_client_cert_key_password, :ssl_client_cert_key) do
+        if values[:ssl_client_cert_key_password] && !values[:ssl_client_cert_key]
+          key(:ssl_client_cert_key).failure(:ssl_client_cert_key_password_with_ssl_client_cert_key)
+        end
+      end
+
+      rule(:ssl_ca_cert) do
+        key.failure(:invalid_certificate) if value && !valid_certificate?(value)
+      end
+
+      rule(:ssl_client_cert) do
+        key.failure(:invalid_certificate) if value && !valid_certificate?(value)
+      end
+
+      rule(:ssl_ca_cert_file_path) do
+        if value
+          if File.exist?(value)
+            key.failure(:invalid_certificate_from_path) unless valid_certificate?(File.read(value))
+          else
+            key.failure(:does_not_exist)
+          end
+        end
+      end
+
+      rule(:ssl_client_cert_key) do
+        key.failure(:invalid_private_key) if value && !valid_private_key?(value)
+      end
+
+      rule(:ssl_client_cert_chain) do
+        key.failure(:invalid_certificate) if value && !valid_certificate?(value)
+      end
+
+      rule(:sasl_oauth_token_provider) do
+        key.failure(:does_not_respond_to_token) if value && !value.respond_to?(:token)
+      end
+
+      rule(:max_wait_time, :socket_timeout) do
+        max_wait_time = values[:max_wait_time]
+        socket_timeout = values[:socket_timeout]
+
+        if socket_timeout.is_a?(Numeric) &&
+           max_wait_time.is_a?(Numeric) &&
+           max_wait_time > socket_timeout
+
+          key(:max_wait_time).failure(:max_wait_time_limit)
+        end
+      end
+
+      rule(:pause_timeout, :pause_max_timeout, :pause_exponential_backoff) do
+        if values[:pause_exponential_backoff]
+          if values[:pause_timeout].to_i > values[:pause_max_timeout].to_i
+            key(:pause_max_timeout).failure(:max_timeout_size_for_exponential)
+          end
+        end
+      end
+
+      private
+
+      # @param value [String] potential RSA key value
+      # @return [Boolean] is the given string a valid RSA key
+      def valid_private_key?(value)
+        OpenSSL::PKey.read(value)
+        true
+      rescue OpenSSL::PKey::PKeyError
+        false
+      end
+
+      # @param value [String] potential X509 cert value
+      # @return [Boolean] is the given string a valid X509 cert
+      def valid_certificate?(value)
+        OpenSSL::X509::Certificate.new(value)
+        true
+      rescue OpenSSL::X509::CertificateError
+        false
+      end
+
+      # @param value [String] potential kafka uri
+      # @return [Boolean] true if it is a kafka uri, otherwise false
+      def kafka_uri?(value)
+        uri = URI.parse(value)
+        URI_SCHEMES.include?(uri.scheme) && uri.port
+      rescue URI::InvalidURIError
+        false
+      end
+    end
+  end
+end

data/lib/karafka/contracts/consumer_group_topic.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Contracts
+    # Consumer group topic validation rules
+    class ConsumerGroupTopic < Dry::Validation::Contract
+      params do
+        required(:id).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)
+        required(:name).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)
+        required(:backend).filled(included_in?: %i[inline sidekiq])
+        required(:consumer).filled
+        required(:deserializer).filled
+        required(:max_bytes_per_partition).filled(:int?, gteq?: 0)
+        required(:start_from_beginning).filled(:bool?)
+        required(:batch_consuming).filled(:bool?)
+      end
+    end
+  end
+end

data/lib/karafka/contracts/responder_usage.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Contracts
+    # Validator to check responder topic usage
+    class ResponderUsageTopic < Dry::Validation::Contract
+      config.messages.load_paths << File.join(Karafka.gem_root, 'config', 'errors.yml')
+
+      params do
+        required(:name).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)
+        required(:required).filled(:bool?)
+        required(:usage_count).filled(:int?, gteq?: 0)
+        required(:registered).filled(eql?: true)
+        required(:async).filled(:bool?)
+        required(:serializer).filled
+      end
+
+      rule(:required, :usage_count) do
+        key(:name).failure(:required_usage_count) if values[:required] && values[:usage_count] < 1
+      end
+    end
+
+    # Validator to check that everything in a responder flow matches responder rules
+    class ResponderUsage < Dry::Validation::Contract
+      include Dry::Core::Constants
+
+      # Contract for verifying the topic usage details
+      TOPIC_CONTRACT = ResponderUsageTopic.new.freeze
+
+      private_constant :TOPIC_CONTRACT
+
+      params do
+        required(:used_topics)
+        required(:registered_topics)
+      end
+
+      rule(:used_topics) do
+        (value || EMPTY_ARRAY).each do |used_topic|
+          TOPIC_CONTRACT.call(used_topic).errors.each do |error|
+            key([:used_topics, used_topic, error.path[0]]).failure(error.text)
+          end
+        end
+      end
+
+      rule(:registered_topics) do
+        (value || EMPTY_ARRAY).each do |used_topic|
+          TOPIC_CONTRACT.call(used_topic).errors.each do |error|
+            key([:registered_topics, used_topic, error.path[0]]).failure(error.text)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/contracts/server_cli_options.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Contracts
+    # Contract for validating correctness of the server cli command options
+    # We validate some basics + the list of consumer_groups on which we want to use, to make
+    # sure that all of them are defined, plus that a pidfile does not exist
+    class ServerCliOptions < Dry::Validation::Contract
+      config.messages.load_paths << File.join(Karafka.gem_root, 'config', 'errors.yml')
+
+      params do
+        optional(:pid).filled(:str?)
+        optional(:daemon).filled(:bool?)
+        optional(:consumer_groups).value(:array, :filled?)
+      end
+
+      rule(:pid) do
+        key(:pid).failure(:pid_already_exists) if value && File.exist?(value)
+      end
+
+      rule(:consumer_groups) do
+        # If there were no consumer_groups declared in the server cli, it means that we will
+        # run all of them and no need to validate them here at all
+        if !value.nil? &&
+           !(value - Karafka::App.config.internal.routing_builder.map(&:name)).empty?
+          key(:consumer_groups).failure(:consumer_groups_inclusion)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/errors.rb

@@ -6,17 +6,18 @@ module Karafka
     # Base class for all the Karafka internal errors
     BaseError = Class.new(StandardError)
 
-    # Should be raised when we attemp to parse incoming params but parsing fails
-    #   If this error (or its descendant) is detected, we will pass the raw message
-    #   into params and proceed further
-    ParserError = Class.new(BaseError)
+    # Should be raised when we have that that we cannot serialize
+    SerializationError = Class.new(BaseError)
+
+    # Should be raised when we tried to deserialize incoming data but we failed
+    DeserializationError = Class.new(BaseError)
 
     # Raised when router receives topic name which does not correspond with any routes
     # This can only happen in a case when:
     #   - you've received a message and we cannot match it with a consumer
     #   - you've changed the routing, so router can no longer associate your topic to
     #     any consumer
-    #   - or in a case when you do a lot of metaprogramming and you change routing/etc on runtime
+    #   - or in a case when you do a lot of meta-programming and you change routing/etc on runtime
     #
     # In case this happens, you will have to create a temporary route that will allow
     # you to "eat" everything from the Sidekiq queue.
@@ -25,29 +26,26 @@ module Karafka
 
     # Raised when we don't use or use responder not in the way it expected to based on the
     # topics usage definitions
-    InvalidResponderUsage = Class.new(BaseError)
+    InvalidResponderUsageError = Class.new(BaseError)
 
-    # Raised when options that we provide to the responder to respond aren't what the schema
+    # Raised when options that we provide to the responder to respond aren't what the contract
     # requires
-    InvalidResponderMessageOptions = Class.new(BaseError)
+    InvalidResponderMessageOptionsError = Class.new(BaseError)
 
-    # Raised when configuration doesn't match with validation schema
-    InvalidConfiguration = Class.new(BaseError)
+    # Raised when configuration doesn't match with validation contract
+    InvalidConfigurationError = Class.new(BaseError)
 
-    # Raised when we try to use Karafka CLI commands (except install) without a bootfile
-    MissingBootFile = Class.new(BaseError)
+    # Raised when we try to use Karafka CLI commands (except install) without a boot file
+    MissingBootFileError = Class.new(BaseError)
 
     # Raised when we want to read a persisted thread messages consumer but it is unavailable
     # This should never happen and if it does, please contact us
-    MissingClient = Class.new(BaseError)
-
-    # Raised when we attemp to pause a partition but the pause timeout is equal to 0
-    InvalidPauseTimeout = Class.new(BaseError)
+    MissingClientError = Class.new(BaseError)
 
     # Raised when want to hook up to an event that is not registered and supported
-    UnregisteredMonitorEvent = Class.new(BaseError)
+    UnregisteredMonitorEventError = Class.new(BaseError)
 
-    # Raised when we've waited enough for shutting down an unresponding process
-    ForcefulShutdown = Class.new(BaseError)
+    # Raised when we've waited enough for shutting down a non-responsive process
+    ForcefulShutdownError = Class.new(BaseError)
   end
 end

data/lib/karafka/fetcher.rb

@@ -5,39 +5,37 @@ module Karafka
   # @note Creating multiple fetchers will result in having multiple connections to the same
   #   topics, which means that if there are no partitions, it won't use them.
   class Fetcher
-    class << self
-      # Starts listening on all the listeners asynchronously
-      # Fetch loop should never end, which means that we won't create more actor clusters
-      # so we don't have to terminate them
-      def call
-        threads = listeners.map do |listener|
-          # We abort on exception because there should be an exception handling developed for
-          # each listener running in separate threads, so the exceptions should never leak
-          # and if that happens, it means that something really bad happened and we should stop
-          # the whole process
-          Thread
-            .new { listener.call }
-            .tap { |thread| thread.abort_on_exception = true }
-        end
-
-        # We aggregate threads here for a supervised shutdown process
-        threads.each { |thread| Karafka::Server.consumer_threads << thread }
-        threads.each(&:join)
-      # If anything crashes here, we need to raise the error and crush the runner because it means
-      # that something terrible happened
-      rescue StandardError => e
-        Karafka.monitor.instrument('fetcher.call.error', caller: self, error: e)
-        Karafka::App.stop!
-        raise e
+    # Starts listening on all the listeners asynchronously
+    # Fetch loop should never end, which means that we won't create more actor clusters
+    # so we don't have to terminate them
+    def call
+      threads = listeners.map do |listener|
+        # We abort on exception because there should be an exception handling developed for
+        # each listener running in separate threads, so the exceptions should never leak
+        # and if that happens, it means that something really bad happened and we should stop
+        # the whole process
+        Thread
+          .new { listener.call }
+          .tap { |thread| thread.abort_on_exception = true }
       end
 
-      private
+      # We aggregate threads here for a supervised shutdown process
+      threads.each { |thread| Karafka::Server.consumer_threads << thread }
+      threads.each(&:join)
+    # If anything crashes here, we need to raise the error and crush the runner because it means
+    # that something terrible happened
+    rescue StandardError => e
+      Karafka.monitor.instrument('fetcher.call.error', caller: self, error: e)
+      Karafka::App.stop!
+      raise e
+    end
+
+    private
 
-      # @return [Array<Karafka::Connection::Listener>] listeners that will consume messages
-      def listeners
-        @listeners ||= App.consumer_groups.active.map do |consumer_group|
-          Karafka::Connection::Listener.new(consumer_group)
-        end
+    # @return [Array<Karafka::Connection::Listener>] listeners that will consume messages
+    def listeners
+      @listeners ||= App.consumer_groups.active.map do |consumer_group|
+        Karafka::Connection::Listener.new(consumer_group)
       end
     end
   end