karafka 1.1.0 → 1.3.0

data/lib/karafka/consumers/callbacks.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Consumers
+    # Additional callbacks that can be used to trigger some actions on certain moments like
+    # manual offset management, committing or anything else outside of a standard messages flow
+    # They are not included by default, as we don't want to provide functionalities that are
+    # not required by users by default
+    # Please refer to the wiki callbacks page for more details on how to use them
+    module Callbacks
+      # Types of events on which we run callbacks
+      TYPES = %i[
+        after_fetch
+        after_poll
+        before_poll
+        before_stop
+      ].freeze
+
+      private_constant :TYPES
+
+      # Class methods needed to make callbacks run
+      module ClassMethods
+        TYPES.each do |type|
+          # Creates a callback wrapper
+          #
+          # @param method_name [Symbol, String] method name or nil if we plan to provide a block
+          # @yield A block with a code that should be executed before scheduling
+          # @note We don't have to optimize the key fetching here as those are class methods that
+          #   are evaluated once upon start
+          define_method(type) do |method_name = nil, &block|
+            key = "consumers.#{Helpers::Inflector.map(to_s)}.#{type}"
+            Karafka::App.monitor.register_event(key)
+
+            Karafka::App.monitor.subscribe(key) do |event|
+              context = event[:context]
+
+              if method_name
+                context.send(method_name)
+              else
+                context.instance_eval(&block)
+              end
+            end
+          end
+        end
+      end
+
+      class << self
+        # @param consumer_class [Class] consumer class that we extend with callbacks
+        def included(consumer_class)
+          consumer_class.class_eval do
+            extend ClassMethods
+          end
+        end
+      end
+
+      # Executes the default consumer flow, runs callbacks and if not halted will call process
+      # method of a proper backend. It is here because it interacts with the default Karafka
+      # call flow and needs to be overwritten to support callbacks
+      def call
+        if self.class.respond_to?(:after_fetch)
+          Karafka::App.monitor.instrument(
+            "consumers.#{Helpers::Inflector.map(self.class.to_s)}.after_fetch",
+            context: self
+          )
+        end
+
+        process
+      end
+    end
+  end
+end

data/lib/karafka/consumers/includer.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Additional functionalities for consumers
+  module Consumers
+    # Module used to inject functionalities into a given consumer instance, based on the consumer
+    # topic and its settings
+    # We don't need all the behaviors in all the cases, so it is not worth having everything
+    # in all the cases all the time
+    module Includer
+      class << self
+        # @param consumer [Karafka::BaseConsumer] consumer instance, that will get some
+        #   functionalities based on the topic under which it operates
+        def call(consumer)
+          topic = consumer.topic
+
+          bind_backend(consumer, topic)
+          bind_params(consumer, topic)
+          bind_metadata(consumer, topic)
+          bind_responders(consumer, topic)
+        end
+
+        private
+
+        # Figures out backend for a given consumer class, based on the topic backend and
+        #   includes it into the consumer class
+        # @param consumer [Karafka::BaseConsumer] consumer instance
+        # @param topic [Karafka::Routing::Topic] topic of a consumer class
+        def bind_backend(consumer, topic)
+          backend = Kernel.const_get("::Karafka::Backends::#{topic.backend.to_s.capitalize}")
+          consumer.extend(backend)
+        end
+
+        # Adds a single #params support for non batch processed topics
+        # @param consumer [Karafka::BaseConsumer] consumer instance
+        # @param topic [Karafka::Routing::Topic] topic of a consumer class
+        def bind_params(consumer, topic)
+          return if topic.batch_consuming
+
+          consumer.extend(SingleParams)
+        end
+
+        # Adds an option to work with metadata for consumer instances that have batch fetching
+        # @param consumer [Karafka::BaseConsumer] consumer instance
+        # @param topic [Karafka::Routing::Topic] topic of a consumer class
+        def bind_metadata(consumer, topic)
+          return unless topic.batch_fetching
+
+          consumer.extend(Metadata)
+        end
+
+        # Adds responders support for topics and consumers with responders defined for them
+        # @param consumer [Karafka::BaseConsumer] consumer instance
+        # @param topic [Karafka::Routing::Topic] topic of a consumer class
+        def bind_responders(consumer, topic)
+          return unless topic.responder
+
+          consumer.extend(Responders)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/consumers/metadata.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Consumers
+    # Brings the metadata into consumers that support batch_fetching
+    module Metadata
+      attr_accessor :metadata
+    end
+  end
+end

data/lib/karafka/consumers/responders.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Consumers
+    # Feature that allows us to use responders flow in consumer
+    module Responders
+      # Responds with given data using given responder. This allows us to have a similar way of
+      # defining flows like synchronous protocols
+      # @param data Anything we want to pass to responder based on which we want to trigger further
+      #   Kafka responding
+      def respond_with(*data)
+        Karafka.monitor.instrument(
+          'consumers.responders.respond_with',
+          caller: self,
+          data: data
+        ) do
+          # @note we build a new instance of responder each time, as a long-running (persisted)
+          #   consumers can respond multiple times during the life-cycle
+          topic.responder.new.call(*data)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/{controllers → consumers}/single_params.rb

@@ -1,12 +1,12 @@
 # frozen_string_literal: true
 
 module Karafka
-  module Controllers
-    # Params alias for single message consumption controllers
+  module Consumers
+    # Params alias for single message consumption consumers
     module SingleParams
       private
 
-      # @return [Karafka::Params::Params] params instance for non batch consumption controllers
+      # @return [Karafka::Params::Params] params instance for non batch consumption consumers
       def params
         params_batch.first
       end

data/lib/karafka/contracts.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace for all the validation contracts that we use to check input
+  module Contracts
+    # Regexp for validating format of groups and topics
+    # @note It is not nested inside of the contracts, as it is used by couple of them
+    TOPIC_REGEXP = /\A(\w|\-|\.)+\z/.freeze
+  end
+end

data/lib/karafka/contracts/config.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Contracts
+    # Contract with validation rules for Karafka configuration details
+    # @note There are many more configuration options inside of the
+    #   Karafka::Setup::Config model, but we don't validate them here as they are
+    #   validated per each route (topic + consumer_group) because they can be overwritten,
+    #   so we validate all of that once all the routes are defined and ready
+    class Config < Dry::Validation::Contract
+      params do
+        required(:client_id).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)
+        required(:shutdown_timeout) { (int? & gt?(0)) }
+        required(:consumer_mapper)
+        required(:topic_mapper)
+
+        optional(:backend).filled
+      end
+    end
+  end
+end

data/lib/karafka/contracts/consumer_group.rb

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Contracts
+    # Contract for single full route (consumer group + topics) validation.
+    class ConsumerGroup < Dry::Validation::Contract
+      config.messages.load_paths << File.join(Karafka.gem_root, 'config', 'errors.yml')
+
+      # Valid uri schemas of Kafka broker url
+      # The ||= is due to the behavior of require_all that resolves dependencies
+      # but sometimes loads things twice
+      URI_SCHEMES ||= %w[kafka kafka+ssl plaintext ssl].freeze
+
+      # Available sasl scram mechanism of authentication (plus nil)
+      SASL_SCRAM_MECHANISMS ||= %w[sha256 sha512].freeze
+
+      # Internal contract for sub-validating topics schema
+      TOPIC_CONTRACT = ConsumerGroupTopic.new.freeze
+
+      private_constant :TOPIC_CONTRACT
+
+      params do
+        required(:id).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)
+        required(:topics).value(:array, :filled?)
+        required(:seed_brokers).value(:array, :filled?)
+        required(:session_timeout).filled { int? | float? }
+        required(:pause_timeout).maybe(%i[integer float]) { filled? > gteq?(0) }
+        required(:pause_max_timeout).maybe(%i[integer float]) { filled? > gteq?(0) }
+        required(:pause_exponential_backoff).filled(:bool?)
+        required(:offset_commit_interval) { int? | float? }
+        required(:offset_commit_threshold).filled(:int?)
+        required(:offset_retention_time).maybe(:integer)
+        required(:heartbeat_interval).filled { (int? | float?) & gteq?(0) }
+        required(:fetcher_max_queue_size).filled(:int?, gt?: 0)
+        required(:connect_timeout).filled { (int? | float?) & gt?(0) }
+        required(:reconnect_timeout).filled { (int? | float?) & gteq?(0) }
+        required(:socket_timeout).filled { (int? | float?) & gt?(0) }
+        required(:min_bytes).filled(:int?, gt?: 0)
+        required(:max_bytes).filled(:int?, gt?: 0)
+        required(:max_wait_time).filled { (int? | float?) & gteq?(0) }
+        required(:batch_fetching).filled(:bool?)
+
+        %i[
+          ssl_ca_cert
+          ssl_ca_cert_file_path
+          ssl_client_cert
+          ssl_client_cert_key
+          ssl_client_cert_chain
+          ssl_client_cert_key_password
+          sasl_gssapi_principal
+          sasl_gssapi_keytab
+          sasl_plain_authzid
+          sasl_plain_username
+          sasl_plain_password
+          sasl_scram_username
+          sasl_scram_password
+        ].each do |encryption_attribute|
+          optional(encryption_attribute).maybe(:str?)
+        end
+
+        optional(:ssl_verify_hostname).maybe(:bool?)
+        optional(:ssl_ca_certs_from_system).maybe(:bool?)
+        optional(:sasl_over_ssl).maybe(:bool?)
+        optional(:sasl_oauth_token_provider).value(:any)
+
+        # It's not with other encryptions as it has some more rules
+        optional(:sasl_scram_mechanism)
+          .maybe(:str?, included_in?: SASL_SCRAM_MECHANISMS)
+      end
+
+      # Uri rule to check if uri is in a Karafka acceptable format
+      rule(:seed_brokers) do
+        if value&.is_a?(Array) && !value.all?(&method(:kafka_uri?))
+          key.failure(:invalid_broker_schema)
+        end
+      end
+
+      rule(:topics) do
+        if value&.is_a?(Array)
+          names = value.map { |topic| topic[:name] }
+
+          key.failure(:topics_names_not_unique) if names.size != names.uniq.size
+        end
+      end
+
+      rule(:topics) do
+        if value&.is_a?(Array)
+          value.each_with_index do |topic, index|
+            TOPIC_CONTRACT.call(topic).errors.each do |error|
+              key([:topics, index, error.path[0]]).failure(error.text)
+            end
+          end
+        end
+      end
+
+      rule(:ssl_client_cert, :ssl_client_cert_key) do
+        if values[:ssl_client_cert] && !values[:ssl_client_cert_key]
+          key(:ssl_client_cert_key).failure(:ssl_client_cert_with_ssl_client_cert_key)
+        end
+      end
+
+      rule(:ssl_client_cert, :ssl_client_cert_key) do
+        if values[:ssl_client_cert_key] && !values[:ssl_client_cert]
+          key(:ssl_client_cert).failure(:ssl_client_cert_key_with_ssl_client_cert)
+        end
+      end
+
+      rule(:ssl_client_cert, :ssl_client_cert_chain) do
+        if values[:ssl_client_cert_chain] && !values[:ssl_client_cert]
+          key(:ssl_client_cert).failure(:ssl_client_cert_chain_with_ssl_client_cert)
+        end
+      end
+
+      rule(:ssl_client_cert_chain, :ssl_client_cert_key) do
+        if values[:ssl_client_cert_chain] && !values[:ssl_client_cert]
+          key(:ssl_client_cert).failure(:ssl_client_cert_chain_with_ssl_client_cert_key)
+        end
+      end
+
+      rule(:ssl_client_cert_key_password, :ssl_client_cert_key) do
+        if values[:ssl_client_cert_key_password] && !values[:ssl_client_cert_key]
+          key(:ssl_client_cert_key).failure(:ssl_client_cert_key_password_with_ssl_client_cert_key)
+        end
+      end
+
+      rule(:ssl_ca_cert) do
+        key.failure(:invalid_certificate) if value && !valid_certificate?(value)
+      end
+
+      rule(:ssl_client_cert) do
+        key.failure(:invalid_certificate) if value && !valid_certificate?(value)
+      end
+
+      rule(:ssl_ca_cert_file_path) do
+        if value
+          if File.exist?(value)
+            key.failure(:invalid_certificate_from_path) unless valid_certificate?(File.read(value))
+          else
+            key.failure(:does_not_exist)
+          end
+        end
+      end
+
+      rule(:ssl_client_cert_key) do
+        key.failure(:invalid_private_key) if value && !valid_private_key?(value)
+      end
+
+      rule(:ssl_client_cert_chain) do
+        key.failure(:invalid_certificate) if value && !valid_certificate?(value)
+      end
+
+      rule(:sasl_oauth_token_provider) do
+        key.failure(:does_not_respond_to_token) if value && !value.respond_to?(:token)
+      end
+
+      rule(:max_wait_time, :socket_timeout) do
+        max_wait_time = values[:max_wait_time]
+        socket_timeout = values[:socket_timeout]
+
+        if socket_timeout.is_a?(Numeric) &&
+           max_wait_time.is_a?(Numeric) &&
+           max_wait_time > socket_timeout
+
+          key(:max_wait_time).failure(:max_wait_time_limit)
+        end
+      end
+
+      rule(:pause_timeout, :pause_max_timeout, :pause_exponential_backoff) do
+        if values[:pause_exponential_backoff]
+          if values[:pause_timeout].to_i > values[:pause_max_timeout].to_i
+            key(:pause_max_timeout).failure(:max_timeout_size_for_exponential)
+          end
+        end
+      end
+
+      private
+
+      # @param value [String] potential RSA key value
+      # @return [Boolean] is the given string a valid RSA key
+      def valid_private_key?(value)
+        OpenSSL::PKey::RSA.new(value)
+        true
+      rescue OpenSSL::PKey::RSAError
+        false
+      end
+
+      # @param value [String] potential X509 cert value
+      # @return [Boolean] is the given string a valid X509 cert
+      def valid_certificate?(value)
+        OpenSSL::X509::Certificate.new(value)
+        true
+      rescue OpenSSL::X509::CertificateError
+        false
+      end
+
+      # @param value [String] potential kafka uri
+      # @return [Boolean] true if it is a kafka uri, otherwise false
+      def kafka_uri?(value)
+        uri = URI.parse(value)
+        URI_SCHEMES.include?(uri.scheme) && uri.port
+      rescue URI::InvalidURIError
+        false
+      end
+    end
+  end
+end

data/lib/karafka/contracts/consumer_group_topic.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Contracts
+    # Consumer group topic validation rules
+    class ConsumerGroupTopic < Dry::Validation::Contract
+      params do
+        required(:id).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)
+        required(:name).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)
+        required(:backend).filled(included_in?: %i[inline sidekiq])
+        required(:consumer).filled
+        required(:deserializer).filled
+        required(:max_bytes_per_partition).filled(:int?, gteq?: 0)
+        required(:start_from_beginning).filled(:bool?)
+        required(:batch_consuming).filled(:bool?)
+      end
+    end
+  end
+end