karafka 1.4.4 → 2.1.10

data/lib/karafka/connection/raw_messages_buffer.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Buffer for raw librdkafka messages.
+    #
+    # When message is added to this buffer, it gets assigned to an array with other messages from
+    # the same topic and partition.
+    #
+    # @note This buffer is NOT threadsafe.
+    #
+    # @note We store data here in groups per topic partition to handle the revocation case, where
+    #    we may need to remove messages from a single topic partition.
+    class RawMessagesBuffer
+      attr_reader :size
+
+      # @return [Karafka::Connection::MessagesBuffer] buffer instance
+      def initialize
+        @size = 0
+        @groups = Hash.new do |topic_groups, topic|
+          topic_groups[topic] = Hash.new do |partition_groups, partition|
+            partition_groups[partition] = []
+          end
+        end
+      end
+
+      # Adds a message to the buffer.
+      #
+      # @param message [Rdkafka::Consumer::Message] raw rdkafka message
+      # @return [Array<Rdkafka::Consumer::Message>] given partition topic sub-buffer array
+      def <<(message)
+        @size += 1
+        @groups[message.topic][message.partition] << message
+      end
+
+      # Allows to iterate over all the topics and partitions messages
+      #
+      # @yieldparam [String] topic name
+      # @yieldparam [Integer] partition number
+      # @yieldparam [Array<Rdkafka::Consumer::Message>] topic partition aggregated results
+      def each
+        @groups.each do |topic, partitions|
+          partitions.each do |partition, messages|
+            yield(topic, partition, messages)
+          end
+        end
+      end
+
+      # Removes given topic and partition data out of the buffer
+      # This is used when there's a partition revocation
+      # @param topic [String] topic we're interested in
+      # @param partition [Integer] partition of which data we want to remove
+      def delete(topic, partition)
+        return unless @groups.key?(topic)
+        return unless @groups.fetch(topic).key?(partition)
+
+        topic_data = @groups.fetch(topic)
+        topic_data.delete(partition)
+
+        recount!
+
+        # If there are no more partitions to handle in a given topic, remove it completely
+        @groups.delete(topic) if topic_data.empty?
+      end
+
+      # Removes duplicated messages from the same partitions
+      # This should be used only when rebalance occurs, as we may get data again we already have
+      # due to the processing from the last offset. In cases like this, we may get same data
+      # again and we do want to ensure as few duplications as possible
+      def uniq!
+        @groups.each_value do |partitions|
+          partitions.each_value do |messages|
+            messages.uniq!(&:offset)
+          end
+        end
+
+        recount!
+      end
+
+      # Removes all the data from the buffer.
+      #
+      # @note We do not clear the whole groups hash but rather we clear the partition hashes, so
+      #   we save ourselves some objects allocations. We cannot clear the underlying arrays as they
+      #   may be used in other threads for data processing, thus if we would clear it, we could
+      #   potentially clear a raw messages array for a job that is in the jobs queue.
+      def clear
+        @size = 0
+        @groups.each_value(&:clear)
+      end
+
+      private
+
+      # Updates the messages count if we performed any operations that could change the state
+      def recount!
+        @size = @groups.each_value.sum do |partitions|
+          partitions.each_value.map(&:count).sum
+        end
+      end
+    end
+  end
+end

data/lib/karafka/connection/rebalance_manager.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Manager for tracking changes in the partitions assignment.
+    #
+    # We need tracking of those to clean up consumers that will no longer process given partitions
+    # as they were taken away.
+    #
+    # @note Since this does not happen really often, we try to stick with same objects for the
+    #   empty states most of the time, so we don't create many objects during the manager life
+    #
+    # @note Internally in the rebalance manager we have a notion of lost partitions. Partitions
+    #   that are lost, are those that got revoked but did not get re-assigned back. We do not
+    #   expose this concept outside and we normalize to have them revoked, as it is irrelevant
+    #   from the rest of the code perspective as only those that are lost are truly revoked.
+    #
+    # @note For cooperative-sticky `#assigned_partitions` holds only the recently assigned
+    #   partitions, not all the partitions that are owned
+    class RebalanceManager
+      # Empty array for internal usage not to create new objects
+      EMPTY_ARRAY = [].freeze
+
+      attr_reader :assigned_partitions, :revoked_partitions
+
+      private_constant :EMPTY_ARRAY
+
+      # @return [RebalanceManager]
+      def initialize
+        @assigned_partitions = {}
+        @revoked_partitions = {}
+        @changed = false
+        @active = false
+      end
+
+      # Resets the rebalance manager state
+      # This needs to be done before each polling loop as during the polling, the state may be
+      # changed
+      def clear
+        @assigned_partitions.clear
+        @revoked_partitions.clear
+        @changed = false
+      end
+
+      # @return [Boolean] indicates a state change in the partitions assignment
+      def changed?
+        @changed
+      end
+
+      # @return [Boolean] true if there was at least one rebalance
+      # @note This method is needed to make sure that when using cooperative-sticky, we do not
+      #   close until first rebalance. Otherwise librdkafka may crash.
+      # @see https://github.com/confluentinc/librdkafka/issues/4312
+      def active?
+        @active
+      end
+
+      # Callback that kicks in inside of rdkafka, when new partitions are assigned.
+      #
+      # @private
+      # @param partitions [Rdkafka::Consumer::TopicPartitionList]
+      def on_partitions_assigned(partitions)
+        @active = true
+        @assigned_partitions = partitions.to_h.transform_values { |part| part.map(&:partition) }
+        @changed = true
+      end
+
+      # Callback that kicks in inside of rdkafka, when partitions are revoked.
+      #
+      # @private
+      # @param partitions [Rdkafka::Consumer::TopicPartitionList]
+      def on_partitions_revoked(partitions)
+        @active = true
+        @revoked_partitions = partitions.to_h.transform_values { |part| part.map(&:partition) }
+        @changed = true
+      end
+
+      # We consider as lost only partitions that were taken away and not re-assigned back to us
+      def lost_partitions
+        lost_partitions = {}
+
+        revoked_partitions.each do |topic, partitions|
+          lost_partitions[topic] = partitions - assigned_partitions.fetch(topic, EMPTY_ARRAY)
+        end
+
+        lost_partitions
+      end
+    end
+  end
+end

data/lib/karafka/contracts/base.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Contracts
+    # Base contract for all Karafka contracts
+    class Base < ::Karafka::Core::Contractable::Contract
+      # @param data [Hash] data for validation
+      # @return [Boolean] true if all good
+      # @raise [Errors::InvalidConfigurationError] invalid configuration error
+      # @note We use contracts only in the config validation context, so no need to add support
+      #   for multiple error classes. It will be added when it will be needed.
+      def validate!(data)
+        super(data, Errors::InvalidConfigurationError)
+      end
+    end
+  end
+end

data/lib/karafka/contracts/config.rb

@@ -2,19 +2,96 @@
 
 module Karafka
   module Contracts
-    # Contract with validation rules for Karafka configuration details
+    # Contract with validation rules for Karafka configuration details.
+    #
     # @note There are many more configuration options inside of the
-    #   Karafka::Setup::Config model, but we don't validate them here as they are
+    #   `Karafka::Setup::Config` model, but we don't validate them here as they are
     #   validated per each route (topic + consumer_group) because they can be overwritten,
-    #   so we validate all of that once all the routes are defined and ready
-    class Config < Dry::Validation::Contract
-      params do
-        required(:client_id).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)
-        required(:shutdown_timeout) { (int? & gt?(0)) }
-        required(:consumer_mapper)
-        required(:topic_mapper)
-
-        optional(:backend).filled
+    #   so we validate all of that once all the routes are defined and ready.
+    class Config < Base
+      configure do |config|
+        config.error_messages = YAML.safe_load(
+          File.read(
+            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+          )
+        ).fetch('en').fetch('validations').fetch('config')
+      end
+
+      # License validity happens in the licenser. Here we do only the simple consistency checks
+      nested(:license) do
+        required(:token) { |val| [true, false].include?(val) || val.is_a?(String) }
+        required(:entity) { |val| val.is_a?(String) }
+      end
+
+      required(:client_id) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
+      required(:concurrency) { |val| val.is_a?(Integer) && val.positive? }
+      required(:consumer_mapper) { |val| !val.nil? }
+      required(:consumer_persistence) { |val| [true, false].include?(val) }
+      required(:pause_timeout) { |val| val.is_a?(Integer) && val.positive? }
+      required(:pause_max_timeout) { |val| val.is_a?(Integer) && val.positive? }
+      required(:pause_with_exponential_backoff) { |val| [true, false].include?(val) }
+      required(:shutdown_timeout) { |val| val.is_a?(Integer) && val.positive? }
+      required(:max_wait_time) { |val| val.is_a?(Integer) && val.positive? }
+      required(:kafka) { |val| val.is_a?(Hash) && !val.empty? }
+
+      # We validate internals just to be sure, that they are present and working
+      nested(:internal) do
+        required(:status) { |val| !val.nil? }
+        required(:process) { |val| !val.nil? }
+
+        nested(:routing) do
+          required(:builder) { |val| !val.nil? }
+          required(:subscription_groups_builder) { |val| !val.nil? }
+        end
+
+        nested(:processing) do
+          required(:jobs_builder) { |val| !val.nil? }
+          required(:scheduler) { |val| !val.nil? }
+          required(:coordinator_class) { |val| !val.nil? }
+          required(:partitioner_class) { |val| !val.nil? }
+        end
+
+        nested(:active_job) do
+          required(:dispatcher) { |val| !val.nil? }
+          required(:job_options_contract) { |val| !val.nil? }
+          required(:consumer_class) { |val| !val.nil? }
+        end
+      end
+
+      virtual do |data, errors|
+        next unless errors.empty?
+
+        detected_errors = []
+
+        data.fetch(:kafka).each_key do |key|
+          next if key.is_a?(Symbol)
+
+          detected_errors << [[:kafka, key], :key_must_be_a_symbol]
+        end
+
+        detected_errors
+      end
+
+      virtual do |data, errors|
+        next unless errors.empty?
+
+        pause_timeout = data.fetch(:pause_timeout)
+        pause_max_timeout = data.fetch(:pause_max_timeout)
+
+        next if pause_timeout <= pause_max_timeout
+
+        [[%i[pause_timeout], :max_timeout_vs_pause_max_timeout]]
+      end
+
+      virtual do |data, errors|
+        next unless errors.empty?
+
+        shutdown_timeout = data.fetch(:shutdown_timeout)
+        max_wait_time = data.fetch(:max_wait_time)
+
+        next if max_wait_time < shutdown_timeout
+
+        [[%i[shutdown_timeout], :shutdown_timeout_vs_max_wait_time]]
       end
     end
   end

data/lib/karafka/contracts/consumer_group.rb

@@ -3,208 +3,53 @@
 module Karafka
   module Contracts
     # Contract for single full route (consumer group + topics) validation.
-    class ConsumerGroup < Dry::Validation::Contract
-      config.messages.load_paths << File.join(Karafka.gem_root, 'config', 'errors.yml')
-
-      # Valid uri schemas of Kafka broker url
-      # The ||= is due to the behavior of require_all that resolves dependencies
-      # but sometimes loads things twice
-      URI_SCHEMES ||= %w[kafka kafka+ssl plaintext ssl].freeze
-
-      # Available sasl scram mechanism of authentication (plus nil)
-      SASL_SCRAM_MECHANISMS ||= %w[sha256 sha512].freeze
-
-      # Internal contract for sub-validating topics schema
-      TOPIC_CONTRACT = ConsumerGroupTopic.new.freeze
-
-      private_constant :TOPIC_CONTRACT
-
-      params do
-        required(:id).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)
-        required(:topics).value(:array, :filled?)
-        required(:seed_brokers).value(:array, :filled?)
-        required(:session_timeout).filled { int? | float? }
-        required(:pause_timeout).maybe(%i[integer float]) { filled? > gteq?(0) }
-        required(:pause_max_timeout).maybe(%i[integer float]) { filled? > gteq?(0) }
-        required(:pause_exponential_backoff).filled(:bool?)
-        required(:offset_commit_interval) { int? | float? }
-        required(:offset_commit_threshold).filled(:int?)
-        required(:offset_retention_time).maybe(:integer)
-        required(:heartbeat_interval).filled { (int? | float?) & gteq?(0) }
-        required(:fetcher_max_queue_size).filled(:int?, gt?: 0)
-        required(:assignment_strategy).value(:any)
-        required(:connect_timeout).filled { (int? | float?) & gt?(0) }
-        required(:reconnect_timeout).filled { (int? | float?) & gteq?(0) }
-        required(:socket_timeout).filled { (int? | float?) & gt?(0) }
-        required(:min_bytes).filled(:int?, gt?: 0)
-        required(:max_bytes).filled(:int?, gt?: 0)
-        required(:max_wait_time).filled { (int? | float?) & gteq?(0) }
-        required(:batch_fetching).filled(:bool?)
-
-        %i[
-          ssl_ca_cert
-          ssl_ca_cert_file_path
-          ssl_client_cert
-          ssl_client_cert_key
-          ssl_client_cert_chain
-          ssl_client_cert_key_password
-          sasl_gssapi_principal
-          sasl_gssapi_keytab
-          sasl_plain_authzid
-          sasl_plain_username
-          sasl_plain_password
-          sasl_scram_username
-          sasl_scram_password
-        ].each do |encryption_attribute|
-          optional(encryption_attribute).maybe(:str?)
-        end
-
-        optional(:ssl_verify_hostname).maybe(:bool?)
-        optional(:ssl_ca_certs_from_system).maybe(:bool?)
-        optional(:sasl_over_ssl).maybe(:bool?)
-        optional(:sasl_oauth_token_provider).value(:any)
-
-        # It's not with other encryptions as it has some more rules
-        optional(:sasl_scram_mechanism)
-          .maybe(:str?, included_in?: SASL_SCRAM_MECHANISMS)
+    class ConsumerGroup < Base
+      configure do |config|
+        config.error_messages = YAML.safe_load(
+          File.read(
+            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+          )
+        ).fetch('en').fetch('validations').fetch('consumer_group')
       end
 
-      # Uri rule to check if uri is in a Karafka acceptable format
-      rule(:seed_brokers) do
-        if value.is_a?(Array) && !value.all?(&method(:kafka_uri?))
-          key.failure(:invalid_broker_schema)
-        end
-      end
+      required(:id) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
+      required(:topics) { |val| val.is_a?(Array) && !val.empty? }
 
-      rule(:topics) do
-        if value.is_a?(Array)
-          names = value.map { |topic| topic[:name] }
+      virtual do |data, errors|
+        next unless errors.empty?
 
-          key.failure(:topics_names_not_unique) if names.size != names.uniq.size
-        end
-      end
+        names = data.fetch(:topics).map { |topic| topic[:name] }
 
-      rule(:topics) do
-        if value.is_a?(Array)
-          value.each_with_index do |topic, index|
-            TOPIC_CONTRACT.call(topic).errors.each do |error|
-              key([:topics, index, error.path[0]]).failure(error.text)
-            end
-          end
-        end
-      end
+        next if names.size == names.uniq.size
 
-      rule(:assignment_strategy) do
-        key.failure(:does_not_respond_to_call) unless value.respond_to?(:call)
+        [[%i[topics], :names_not_unique]]
       end
 
-      rule(:ssl_client_cert, :ssl_client_cert_key) do
-        if values[:ssl_client_cert] && !values[:ssl_client_cert_key]
-          key(:ssl_client_cert_key).failure(:ssl_client_cert_with_ssl_client_cert_key)
-        end
-      end
+      virtual do |data, errors|
+        next unless errors.empty?
+        next unless ::Karafka::App.config.strict_topics_namespacing
 
-      rule(:ssl_client_cert, :ssl_client_cert_key) do
-        if values[:ssl_client_cert_key] && !values[:ssl_client_cert]
-          key(:ssl_client_cert).failure(:ssl_client_cert_key_with_ssl_client_cert)
-        end
-      end
+        names = data.fetch(:topics).map { |topic| topic[:name] }
+        names_hash = names.each_with_object({}) { |n, h| h[n] = true }
+        error_occured = false
+        names.each do |n|
+          # Skip topic names that are not namespaced
+          next unless n.chars.find { |c| ['.', '_'].include?(c) }
 
-      rule(:ssl_client_cert, :ssl_client_cert_chain) do
-        if values[:ssl_client_cert_chain] && !values[:ssl_client_cert]
-          key(:ssl_client_cert).failure(:ssl_client_cert_chain_with_ssl_client_cert)
-        end
-      end
-
-      rule(:ssl_client_cert_chain, :ssl_client_cert_key) do
-        if values[:ssl_client_cert_chain] && !values[:ssl_client_cert]
-          key(:ssl_client_cert).failure(:ssl_client_cert_chain_with_ssl_client_cert_key)
-        end
-      end
-
-      rule(:ssl_client_cert_key_password, :ssl_client_cert_key) do
-        if values[:ssl_client_cert_key_password] && !values[:ssl_client_cert_key]
-          key(:ssl_client_cert_key).failure(:ssl_client_cert_key_password_with_ssl_client_cert_key)
-        end
-      end
-
-      rule(:ssl_ca_cert) do
-        key.failure(:invalid_certificate) if value && !valid_certificate?(value)
-      end
-
-      rule(:ssl_client_cert) do
-        key.failure(:invalid_certificate) if value && !valid_certificate?(value)
-      end
-
-      rule(:ssl_ca_cert_file_path) do
-        if value
-          if File.exist?(value)
-            key.failure(:invalid_certificate_from_path) unless valid_certificate?(File.read(value))
+          if n.chars.include?('.')
+            # Check underscore styled topic
+            underscored_topic = n.tr('.', '_')
+            error_occured = names_hash[underscored_topic] ? true : false
           else
-            key.failure(:does_not_exist)
-          end
-        end
-      end
-
-      rule(:ssl_client_cert_key) do
-        key.failure(:invalid_private_key) if value && !valid_private_key?(value)
-      end
-
-      rule(:ssl_client_cert_chain) do
-        key.failure(:invalid_certificate) if value && !valid_certificate?(value)
-      end
-
-      rule(:sasl_oauth_token_provider) do
-        key.failure(:does_not_respond_to_token) if value && !value.respond_to?(:token)
-      end
-
-      rule(:max_wait_time, :socket_timeout) do
-        max_wait_time = values[:max_wait_time]
-        socket_timeout = values[:socket_timeout]
-
-        if socket_timeout.is_a?(Numeric) &&
-           max_wait_time.is_a?(Numeric) &&
-           max_wait_time > socket_timeout
-
-          key(:max_wait_time).failure(:max_wait_time_limit)
-        end
-      end
-
-      rule(:pause_timeout, :pause_max_timeout, :pause_exponential_backoff) do
-        if values[:pause_exponential_backoff]
-          if values[:pause_timeout].to_i > values[:pause_max_timeout].to_i
-            key(:pause_max_timeout).failure(:max_timeout_size_for_exponential)
+            # Check dot styled topic
+            dot_topic = n.tr('_', '.')
+            error_occured = names_hash[dot_topic] ? true : false
           end
         end
-      end
-
-      private
 
-      # @param value [String] potential RSA key value
-      # @return [Boolean] is the given string a valid RSA key
-      def valid_private_key?(value)
-        OpenSSL::PKey.read(value)
-        true
-      rescue OpenSSL::PKey::PKeyError
-        false
-      end
-
-      # @param value [String] potential X509 cert value
-      # @return [Boolean] is the given string a valid X509 cert
-      def valid_certificate?(value)
-        OpenSSL::X509::Certificate.new(value)
-        true
-      rescue OpenSSL::X509::CertificateError
-        false
-      end
+        next unless error_occured
 
-      # @param value [String] potential kafka uri
-      # @return [Boolean] true if it is a kafka uri, otherwise false
-      def kafka_uri?(value)
-        uri = URI.parse(value)
-        URI_SCHEMES.include?(uri.scheme) && uri.port
-      rescue URI::InvalidURIError
-        false
+        [[%i[topics], :topics_namespaced_names_not_unique]]
       end
     end
   end

data/lib/karafka/contracts/server_cli_options.rb

@@ -2,29 +2,90 @@
 
 module Karafka
   module Contracts
-    # Contract for validating correctness of the server cli command options
-    # We validate some basics + the list of consumer_groups on which we want to use, to make
-    # sure that all of them are defined, plus that a pidfile does not exist
-    class ServerCliOptions < Dry::Validation::Contract
-      config.messages.load_paths << File.join(Karafka.gem_root, 'config', 'errors.yml')
-
-      params do
-        optional(:pid).filled(:str?)
-        optional(:daemon).filled(:bool?)
-        optional(:consumer_groups).value(:array, :filled?)
+    # Contract for validating correctness of the server cli command options.
+    class ServerCliOptions < Base
+      configure do |config|
+        config.error_messages = YAML.safe_load(
+          File.read(
+            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+          )
+        ).fetch('en').fetch('validations').fetch('server_cli_options')
       end
 
-      rule(:pid) do
-        key(:pid).failure(:pid_already_exists) if value && File.exist?(value)
-      end
+      %i[
+        include
+        exclude
+      ].each do |action|
+        optional(:"#{action}_consumer_groups") { |cg| cg.is_a?(Array) }
+        optional(:"#{action}_subscription_groups") { |sg| sg.is_a?(Array) }
+        optional(:"#{action}_topics") { |topics| topics.is_a?(Array) }
+
+        virtual do |data, errors|
+          next unless errors.empty?
+
+          value = data.fetch(:"#{action}_consumer_groups")
+
+          # If there were no consumer_groups declared in the server cli, it means that we will
+          # run all of them and no need to validate them here at all
+          next if value.empty?
+          next if (value - Karafka::App.consumer_groups.map(&:name)).empty?
+
+          # Found unknown consumer groups
+          [[[:"#{action}_consumer_groups"], :consumer_groups_inclusion]]
+        end
+
+        virtual do |data, errors|
+          next unless errors.empty?
+
+          value = data.fetch(:"#{action}_subscription_groups")
+
+          # If there were no subscription_groups declared in the server cli, it means that we will
+          # run all of them and no need to validate them here at all
+          next if value.empty?
+
+          subscription_groups = Karafka::App
+                                .consumer_groups
+                                .map(&:subscription_groups)
+                                .flatten
+                                .map(&:name)
+
+          next if (value - subscription_groups).empty?
 
-      rule(:consumer_groups) do
-        # If there were no consumer_groups declared in the server cli, it means that we will
-        # run all of them and no need to validate them here at all
-        if !value.nil? &&
-           !(value - Karafka::App.config.internal.routing_builder.map(&:name)).empty?
-          key(:consumer_groups).failure(:consumer_groups_inclusion)
+          # Found unknown subscription groups
+          [[[:"#{action}_subscription_groups"], :subscription_groups_inclusion]]
         end
+
+        virtual do |data, errors|
+          next unless errors.empty?
+
+          value = data.fetch(:"#{action}_topics")
+
+          # If there were no topics declared in the server cli, it means that we will
+          # run all of them and no need to validate them here at all
+          next if value.empty?
+
+          topics = Karafka::App
+                   .consumer_groups
+                   .map(&:subscription_groups)
+                   .flatten
+                   .map(&:topics)
+                   .map { |gtopics| gtopics.map(&:name) }
+                   .flatten
+
+          next if (value - topics).empty?
+
+          # Found unknown topics
+          [[[:"#{action}_topics"], :topics_inclusion]]
+        end
+      end
+
+      # Makes sure we have anything to subscribe to when we start the server
+      virtual do |_, errors|
+        next unless errors.empty?
+
+        next unless Karafka::App.subscription_groups.empty?
+
+        [[%i[include_topics], :topics_missing]]
       end
     end
   end