karafka 2.5.0.rc2 → 2.5.1.beta1

data/lib/karafka/contracts/config.rb

@@ -1,210 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Contracts
-    # Contract with validation rules for Karafka configuration details.
-    #
-    # @note There are many more configuration options inside of the
-    #   `Karafka::Setup::Config` model, but we don't validate them here as they are
-    #   validated per each route (topic + consumer_group) because they can be overwritten,
-    #   so we validate all of that once all the routes are defined and ready.
-    class Config < Base
-      configure do |config|
-        config.error_messages = YAML.safe_load(
-          File.read(
-            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
-          )
-        ).fetch('en').fetch('validations').fetch('config')
-      end
-
-      # License validity happens in the licenser. Here we do only the simple consistency checks
-      nested(:license) do
-        required(:token) { |val| [true, false].include?(val) || val.is_a?(String) }
-        required(:entity) { |val| val.is_a?(String) }
-      end
-
-      required(:client_id) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
-      required(:concurrency) { |val| val.is_a?(Integer) && val.positive? }
-      required(:consumer_persistence) { |val| [true, false].include?(val) }
-      required(:pause_timeout) { |val| val.is_a?(Integer) && val.positive? }
-      required(:pause_max_timeout) { |val| val.is_a?(Integer) && val.positive? }
-      required(:pause_with_exponential_backoff) { |val| [true, false].include?(val) }
-      required(:strict_topics_namespacing) { |val| [true, false].include?(val) }
-      required(:shutdown_timeout) { |val| val.is_a?(Integer) && val.positive? }
-      required(:max_wait_time) { |val| val.is_a?(Integer) && val.positive? }
-      required(:group_id) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
-      required(:kafka) { |val| val.is_a?(Hash) && !val.empty? }
-      required(:strict_declarative_topics) { |val| [true, false].include?(val) }
-      required(:worker_thread_priority) { |val| (-3..3).to_a.include?(val) }
-
-      nested(:swarm) do
-        required(:nodes) { |val| val.is_a?(Integer) && val.positive? }
-        required(:node) { |val| val == false || val.is_a?(Karafka::Swarm::Node) }
-      end
-
-      nested(:oauth) do
-        required(:token_provider_listener) do |val|
-          val == false || val.respond_to?(:on_oauthbearer_token_refresh)
-        end
-      end
-
-      nested(:admin) do
-        # Can be empty because inherits values from the root kafka
-        required(:kafka) { |val| val.is_a?(Hash) }
-        required(:group_id) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
-        required(:max_wait_time) { |val| val.is_a?(Integer) && val.positive? }
-        required(:retry_backoff) { |val| val.is_a?(Integer) && val >= 100 }
-        required(:max_retries_duration) { |val| val.is_a?(Integer) && val >= 1_000 }
-      end
-
-      # We validate internals just to be sure, that they are present and working
-      nested(:internal) do
-        required(:status) { |val| !val.nil? }
-        required(:process) { |val| !val.nil? }
-        # In theory this could be less than a second, however this would impact the maximum time
-        # of a single consumer queue poll, hence we prevent it
-        required(:tick_interval) { |val| val.is_a?(Integer) && val >= 1_000 }
-        required(:supervision_sleep) { |val| val.is_a?(Numeric) && val.positive? }
-        required(:forceful_exit_code) { |val| val.is_a?(Integer) && val >= 0 }
-
-        nested(:swarm) do
-          required(:manager) { |val| !val.nil? }
-          required(:orphaned_exit_code) { |val| val.is_a?(Integer) && val >= 0 }
-          required(:pidfd_open_syscall) { |val| val.is_a?(Integer) && val >= 0 }
-          required(:pidfd_signal_syscall) { |val| val.is_a?(Integer) && val >= 0 }
-          required(:supervision_interval) { |val| val.is_a?(Integer) && val >= 1_000 }
-          required(:liveness_interval) { |val| val.is_a?(Integer) && val >= 1_000 }
-          required(:liveness_listener) { |val| !val.nil? }
-          required(:node_report_timeout) { |val| val.is_a?(Integer) && val >= 1_000 }
-          required(:node_restart_timeout) { |val| val.is_a?(Integer) && val >= 1_000 }
-        end
-
-        nested(:connection) do
-          required(:manager) { |val| !val.nil? }
-          required(:conductor) { |val| !val.nil? }
-          required(:reset_backoff) { |val| val.is_a?(Integer) && val >= 1_000 }
-          required(:listener_thread_priority) { |val| (-3..3).to_a.include?(val) }
-
-          nested(:proxy) do
-            nested(:commit) do
-              required(:max_attempts) { |val| val.is_a?(Integer) && val.positive? }
-              required(:wait_time) { |val| val.is_a?(Integer) && val.positive? }
-            end
-
-            # All of them have the same requirements
-            %i[
-              query_watermark_offsets
-              offsets_for_times
-              committed
-              metadata
-            ].each do |scope|
-              nested(scope) do
-                required(:timeout) { |val| val.is_a?(Integer) && val.positive? }
-                required(:max_attempts) { |val| val.is_a?(Integer) && val.positive? }
-                required(:wait_time) { |val| val.is_a?(Integer) && val.positive? }
-              end
-            end
-          end
-        end
-
-        nested(:routing) do
-          required(:builder) { |val| !val.nil? }
-          required(:subscription_groups_builder) { |val| !val.nil? }
-        end
-
-        nested(:processing) do
-          required(:jobs_builder) { |val| !val.nil? }
-          required(:jobs_queue_class) { |val| !val.nil? }
-          required(:scheduler_class) { |val| !val.nil? }
-          required(:coordinator_class) { |val| !val.nil? }
-          required(:errors_tracker_class) { |val| val.nil? || val.is_a?(Class) }
-          required(:partitioner_class) { |val| !val.nil? }
-          required(:strategy_selector) { |val| !val.nil? }
-          required(:expansions_selector) { |val| !val.nil? }
-          required(:executor_class) { |val| !val.nil? }
-          required(:worker_job_call_wrapper) { |val| val == false || val.respond_to?(:wrap) }
-        end
-
-        nested(:active_job) do
-          required(:dispatcher) { |val| !val.nil? }
-          required(:job_options_contract) { |val| !val.nil? }
-          required(:consumer_class) { |val| !val.nil? }
-        end
-      end
-
-      # Ensure all root kafka keys are symbols
-      virtual do |data, errors|
-        next unless errors.empty?
-
-        detected_errors = []
-
-        data.fetch(:kafka).each_key do |key|
-          next if key.is_a?(Symbol)
-
-          detected_errors << [[:kafka, key], :key_must_be_a_symbol]
-        end
-
-        detected_errors
-      end
-
-      # Ensure all admin kafka keys are symbols
-      virtual do |data, errors|
-        next unless errors.empty?
-
-        detected_errors = []
-
-        data.fetch(:admin).fetch(:kafka).each_key do |key|
-          next if key.is_a?(Symbol)
-
-          detected_errors << [[:admin, :kafka, key], :key_must_be_a_symbol]
-        end
-
-        detected_errors
-      end
-
-      virtual do |data, errors|
-        next unless errors.empty?
-
-        pause_timeout = data.fetch(:pause_timeout)
-        pause_max_timeout = data.fetch(:pause_max_timeout)
-
-        next if pause_timeout <= pause_max_timeout
-
-        [[%i[pause_timeout], :max_timeout_vs_pause_max_timeout]]
-      end
-
-      virtual do |data, errors|
-        next unless errors.empty?
-
-        shutdown_timeout = data.fetch(:shutdown_timeout)
-        max_wait_time = data.fetch(:max_wait_time)
-
-        next if max_wait_time < shutdown_timeout
-
-        [[%i[shutdown_timeout], :shutdown_timeout_vs_max_wait_time]]
-      end
-
-      # `internal.swarm.node_report_timeout` should not be close to `max_wait_time` otherwise
-      # there may be a case where node cannot report often enough because it is clogged by waiting
-      # on more data.
-      #
-      # We handle that at a config level to make sure that this is correctly configured.
-      #
-      # We do not validate this in the context of swarm usage (validate only if...) because it is
-      # often that swarm only runs on prod and we do not want to crash it surprisingly.
-      virtual do |data, errors|
-        next unless errors.empty?
-
-        max_wait_time = data.fetch(:max_wait_time)
-        node_report_timeout = data.fetch(:internal)[:swarm][:node_report_timeout] || false
-
-        next unless node_report_timeout
-        # max wait time should be at least 20% smaller than the reporting time to have enough
-        # time for reporting
-        next if max_wait_time < node_report_timeout * 0.8
-
-        [[%i[max_wait_time], :max_wait_time_vs_swarm_node_report_timeout]]
-      end
-    end
-  end
-end

data/lib/karafka/contracts/consumer_group.rb

@@ -1,81 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Contracts
-    # Contract for single full route (consumer group + topics) validation.
-    class ConsumerGroup < Base
-      configure do |config|
-        config.error_messages = YAML.safe_load(
-          File.read(
-            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
-          )
-        ).fetch('en').fetch('validations').fetch('consumer_group')
-      end
-
-      required(:id) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
-      required(:topics) { |val| val.is_a?(Array) && !val.empty? }
-
-      virtual do |data, errors|
-        next unless errors.empty?
-
-        names = data.fetch(:topics).map { |topic| topic_unique_key(topic) }
-
-        next if names.size == names.uniq.size
-
-        [[%i[topics], :names_not_unique]]
-      end
-
-      # Prevent same topics subscriptions in one CG with different consumer classes
-      # This should prevent users from accidentally creating multi-sg one CG setup with weird
-      # different consumer usage. If you need to consume same topic twice, use distinct CGs.
-      virtual do |data, errors|
-        next unless errors.empty?
-
-        topics_consumers = Hash.new { |h, k| h[k] = Set.new }
-
-        data.fetch(:topics).map do |topic|
-          topics_consumers[topic[:name]] << topic[:consumer]
-        end
-
-        next if topics_consumers.values.map(&:size).all? { |count| count == 1 }
-
-        [[%i[topics], :many_consumers_same_topic]]
-      end
-
-      virtual do |data, errors|
-        next unless errors.empty?
-        next unless ::Karafka::App.config.strict_topics_namespacing
-
-        names = data.fetch(:topics).map { |topic| topic[:name] }
-        names_hash = names.each_with_object({}) { |n, h| h[n] = true }
-        error_occured = false
-        names.each do |n|
-          # Skip topic names that are not namespaced
-          next unless n.chars.find { |c| ['.', '_'].include?(c) }
-
-          if n.chars.include?('.')
-            # Check underscore styled topic
-            underscored_topic = n.tr('.', '_')
-            error_occured = names_hash[underscored_topic] ? true : false
-          else
-            # Check dot styled topic
-            dot_topic = n.tr('_', '.')
-            error_occured = names_hash[dot_topic] ? true : false
-          end
-        end
-
-        next unless error_occured
-
-        [[%i[topics], :topics_namespaced_names_not_unique]]
-      end
-
-      class << self
-        # @param topic [Hash] topic config hash
-        # @return [String] topic unique key for validators
-        def topic_unique_key(topic)
-          topic[:name]
-        end
-      end
-    end
-  end
-end

data/lib/karafka/contracts/routing.rb

@@ -1,59 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Contracts
-    # Ensures that routing wide rules are obeyed
-    class Routing < Base
-      configure do |config|
-        config.error_messages = YAML.safe_load(
-          File.read(
-            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
-          )
-        ).fetch('en').fetch('validations').fetch('routing')
-      end
-
-      # Ensures, that when declarative topics strict requirement is on, all topics have
-      # declarative definition (including DLQ topics)
-      # @note It will ignore routing pattern topics because those topics are virtual
-      virtual do |data, errors|
-        next unless errors.empty?
-        # Do not validate declaratives unless required and explicitly enabled
-        next unless Karafka::App.config.strict_declarative_topics
-
-        # Collects declarative topics. Please note, that any topic that has a `#topic` reference,
-        # will be declarative by default unless explicitly disabled. This however does not apply
-        # to the DLQ definitions
-        dec_topics = Set.new
-        # All topics including the DLQ topics names that are marked as active
-        topics = Set.new
-
-        data.each do |consumer_group|
-          consumer_group[:topics].each do |topic|
-            pat = topic[:patterns]
-            # Ignore pattern topics because they won't exist and should not be declarative
-            # managed
-            topics << topic[:name] if !pat || !pat[:active]
-
-            dlq = topic[:dead_letter_queue]
-            topics << dlq[:topic] if dlq[:active]
-
-            dec = topic[:declaratives]
-
-            dec_topics << topic[:name] if dec[:active]
-          end
-        end
-
-        missing_dec = topics - dec_topics
-
-        next if missing_dec.empty?
-
-        missing_dec.map do |topic_name|
-          [
-            [:topics, topic_name],
-            :without_declarative_definition
-          ]
-        end
-      end
-    end
-  end
-end

data/lib/karafka/contracts/server_cli_options.rb

@@ -1,92 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Contracts
-    # Contract for validating correctness of the server cli command options.
-    class ServerCliOptions < Base
-      configure do |config|
-        config.error_messages = YAML.safe_load(
-          File.read(
-            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
-          )
-        ).fetch('en').fetch('validations').fetch('server_cli_options')
-      end
-
-      %i[
-        include
-        exclude
-      ].each do |action|
-        optional(:"#{action}_consumer_groups") { |cg| cg.is_a?(Array) }
-        optional(:"#{action}_subscription_groups") { |sg| sg.is_a?(Array) }
-        optional(:"#{action}_topics") { |topics| topics.is_a?(Array) }
-
-        virtual do |data, errors|
-          next unless errors.empty?
-
-          value = data.fetch(:"#{action}_consumer_groups")
-
-          # If there were no consumer_groups declared in the server cli, it means that we will
-          # run all of them and no need to validate them here at all
-          next if value.empty?
-          next if (value - Karafka::App.consumer_groups.map(&:name)).empty?
-
-          # Found unknown consumer groups
-          [[[:"#{action}_consumer_groups"], :consumer_groups_inclusion]]
-        end
-
-        virtual do |data, errors|
-          next unless errors.empty?
-
-          value = data.fetch(:"#{action}_subscription_groups")
-
-          # If there were no subscription_groups declared in the server cli, it means that we will
-          # run all of them and no need to validate them here at all
-          next if value.empty?
-
-          subscription_groups = Karafka::App
-                                .consumer_groups
-                                .map(&:subscription_groups)
-                                .flatten
-                                .map(&:name)
-
-          next if (value - subscription_groups).empty?
-
-          # Found unknown subscription groups
-          [[[:"#{action}_subscription_groups"], :subscription_groups_inclusion]]
-        end
-
-        virtual do |data, errors|
-          next unless errors.empty?
-
-          value = data.fetch(:"#{action}_topics")
-
-          # If there were no topics declared in the server cli, it means that we will
-          # run all of them and no need to validate them here at all
-          next if value.empty?
-
-          topics = Karafka::App
-                   .consumer_groups
-                   .map(&:subscription_groups)
-                   .flatten
-                   .map(&:topics)
-                   .map { |gtopics| gtopics.map(&:name) }
-                   .flatten
-
-          next if (value - topics).empty?
-
-          # Found unknown topics
-          [[[:"#{action}_topics"], :topics_inclusion]]
-        end
-      end
-
-      # Makes sure we have anything to subscribe to when we start the server
-      virtual do |_, errors|
-        next unless errors.empty?
-
-        next unless Karafka::App.subscription_groups.empty?
-
-        [[%i[include_topics], :topics_missing]]
-      end
-    end
-  end
-end

data/lib/karafka/contracts/topic.rb

@@ -1,81 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Contracts
-    # Consumer group topic validation rules.
-    class Topic < Base
-      configure do |config|
-        config.error_messages = YAML.safe_load(
-          File.read(
-            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
-          )
-        ).fetch('en').fetch('validations').fetch('topic')
-      end
-
-      required(:deserializers) { |val| !val.nil? }
-      required(:id) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
-      required(:kafka) { |val| val.is_a?(Hash) && !val.empty? }
-      required(:max_messages) { |val| val.is_a?(Integer) && val >= 1 }
-      required(:initial_offset) { |val| %w[earliest latest].include?(val) }
-      required(:max_wait_time) { |val| val.is_a?(Integer) && val >= 10 }
-      required(:name) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
-      required(:active) { |val| [true, false].include?(val) }
-      nested(:subscription_group_details) do
-        required(:name) { |val| val.is_a?(String) && !val.empty? }
-      end
-
-      # Consumer needs to be present only if topic is active
-      # We allow not to define consumer for non-active because they may be only used via admin
-      # api or other ways and not consumed with consumer
-      virtual do |data, errors|
-        next unless errors.empty?
-        next if data.fetch(:consumer)
-        next unless data.fetch(:active)
-
-        [[%w[consumer], :missing]]
-      end
-
-      virtual do |data, errors|
-        next unless errors.empty?
-
-        value = data.fetch(:kafka)
-
-        begin
-          # This will trigger rdkafka validations that we catch and re-map the info and use dry
-          # compatible format
-          Rdkafka::Config.new(value).send(:native_config)
-
-          nil
-        rescue Rdkafka::Config::ConfigError => e
-          [[%w[kafka], e.message]]
-        end
-      end
-
-      # When users redefine kafka scope settings per topic, they often forget to define the
-      # basic stuff as they assume it is auto-inherited. It is not (unless inherit flag used),
-      # leaving them with things like bootstrap.servers undefined. This checks that bootstrap
-      # servers are defined so we can catch those issues before they cause more problems.
-      virtual do |data, errors|
-        next unless errors.empty?
-
-        kafka = data.fetch(:kafka)
-
-        next if kafka.key?(:'bootstrap.servers')
-
-        [[%w[kafka bootstrap.servers], :missing]]
-      end
-
-      virtual do |data, errors|
-        next unless errors.empty?
-        next unless ::Karafka::App.config.strict_topics_namespacing
-
-        value = data.fetch(:name)
-        namespacing_chars_count = value.chars.find_all { |c| ['.', '_'].include?(c) }.uniq.size
-
-        next if namespacing_chars_count <= 1
-
-        [[%w[name], :inconsistent_namespacing]]
-      end
-    end
-  end
-end

data/lib/karafka/swarm/pidfd.rb

@@ -1,147 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Swarm
-    # Pidfd Linux representation wrapped with Ruby for communication within Swarm
-    # It is more stable than using `#pid` and `#ppid` + signals and cheaper
-    class Pidfd
-      include Helpers::ConfigImporter.new(
-        pidfd_open_syscall: %i[internal swarm pidfd_open_syscall],
-        pidfd_signal_syscall: %i[internal swarm pidfd_signal_syscall],
-        waitid_syscall: %i[internal swarm waitid_syscall]
-      )
-
-      extend FFI::Library
-
-      begin
-        ffi_lib FFI::Library::LIBC
-
-        # direct usage of this is only available since glibc 2.36, hence we use bindings and call
-        # it directly via syscalls
-        attach_function :fdpid_open, :syscall, %i[long int uint], :int
-        attach_function :fdpid_signal, :syscall, %i[long int int pointer uint], :int
-        attach_function :waitid, %i[int int pointer uint], :int
-
-        API_SUPPORTED = true
-      # LoadError is a parent to FFI::NotFoundError
-      rescue LoadError
-        API_SUPPORTED = false
-      ensure
-        private_constant :API_SUPPORTED
-      end
-
-      # https://github.com/torvalds/linux/blob/7e90b5c295/include/uapi/linux/wait.h#L20
-      P_PIDFD = 3
-
-      # Wait for child processes that have exited
-      WEXITED = 4
-
-      private_constant :P_PIDFD, :WEXITED
-
-      class << self
-        # @return [Boolean] true if syscall is supported via FFI
-        def supported?
-          # If we were not even able to load the FFI C lib, it won't be supported
-          return false unless API_SUPPORTED
-          # Won't work on macOS because it does not support pidfd
-          return false if RUBY_DESCRIPTION.include?('darwin')
-          # Won't work on Windows for the same reason as on macOS
-          return false if RUBY_DESCRIPTION.match?(/mswin|ming|cygwin/)
-
-          # There are some OSes like BSD that will have C lib for FFI bindings but will not support
-          # the needed syscalls. In such cases, we can just try and fail, which will indicate it
-          # won't work. The same applies to using new glibc on an old kernel.
-          new(::Process.pid)
-
-          true
-        rescue Errors::PidfdOpenFailedError
-          false
-        end
-      end
-
-      # @param pid [Integer] pid of the node we want to work with
-      def initialize(pid)
-        @mutex = Mutex.new
-
-        @pid = pid
-        @pidfd = open(pid)
-        @pidfd_io = IO.new(@pidfd)
-      end
-
-      # @return [Boolean] true if given process is alive, false if no longer
-      def alive?
-        @pidfd_select ||= [@pidfd_io]
-
-        if @mutex.owned?
-          return false if @cleaned
-
-          IO.select(@pidfd_select, nil, nil, 0).nil?
-        else
-          @mutex.synchronize do
-            return false if @cleaned
-
-            IO.select(@pidfd_select, nil, nil, 0).nil?
-          end
-        end
-      end
-
-      # Cleans the zombie process
-      # @note This should run **only** on processes that exited, otherwise will wait
-      def cleanup
-        @mutex.synchronize do
-          return if @cleaned
-
-          waitid(P_PIDFD, @pidfd, nil, WEXITED)
-
-          @pidfd_io.close
-          @pidfd_select = nil
-          @pidfd_io = nil
-          @pidfd = nil
-          @cleaned = true
-        end
-      end
-
-      # Sends given signal to the process using its pidfd
-      # @param sig_name [String] signal name
-      # @return [Boolean] true if signal was sent, otherwise false or error raised. `false`
-      #   returned when we attempt to send a signal to a dead process
-      # @note It will not send signals to dead processes
-      def signal(sig_name)
-        @mutex.synchronize do
-          return false if @cleaned
-          # Never signal processes that are dead
-          return false unless alive?
-
-          result = fdpid_signal(
-            pidfd_signal_syscall,
-            @pidfd,
-            Signal.list.fetch(sig_name),
-            nil,
-            0
-          )
-
-          return true if result.zero?
-
-          raise Errors::PidfdSignalFailedError, result
-        end
-      end
-
-      private
-
-      # Opens a pidfd for the provided pid
-      # @param pid [Integer]
-      # @return [Integer] pidfd
-      def open(pid)
-        pidfd = fdpid_open(
-          pidfd_open_syscall,
-          pid,
-          0
-        )
-
-        return pidfd if pidfd != -1
-
-        raise Errors::PidfdOpenFailedError, pidfd
-      end
-    end
-  end
-end