karafka 2.5.0 → 2.5.1.beta1

data/lib/karafka/routing/contracts/topic.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Contracts
+      # Consumer group topic validation rules.
+      class Topic < Karafka::Contracts::Base
+        configure do |config|
+          config.error_messages = YAML.safe_load(
+            File.read(
+              File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+            )
+          ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
+        end
+
+        required(:deserializers) { |val| !val.nil? }
+        required(:id) { |val| val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val) }
+        required(:kafka) { |val| val.is_a?(Hash) && !val.empty? }
+        required(:max_messages) { |val| val.is_a?(Integer) && val >= 1 }
+        required(:initial_offset) { |val| %w[earliest latest].include?(val) }
+        required(:max_wait_time) { |val| val.is_a?(Integer) && val >= 10 }
+        required(:name) { |val| val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val) }
+        required(:active) { |val| [true, false].include?(val) }
+        nested(:subscription_group_details) do
+          required(:name) { |val| val.is_a?(String) && !val.empty? }
+        end
+
+        # Consumer needs to be present only if topic is active
+        # We allow not to define consumer for non-active because they may be only used via admin
+        # api or other ways and not consumed with consumer
+        virtual do |data, errors|
+          next unless errors.empty?
+          next if data.fetch(:consumer)
+          next unless data.fetch(:active)
+
+          [[%w[consumer], :missing]]
+        end
+
+        virtual do |data, errors|
+          next unless errors.empty?
+
+          value = data.fetch(:kafka)
+
+          begin
+            # This will trigger rdkafka validations that we catch and re-map the info and use dry
+            # compatible format
+            Rdkafka::Config.new(value).send(:native_config)
+
+            nil
+          rescue Rdkafka::Config::ConfigError => e
+            [[%w[kafka], e.message]]
+          end
+        end
+
+        # When users redefine kafka scope settings per topic, they often forget to define the
+        # basic stuff as they assume it is auto-inherited. It is not (unless inherit flag used),
+        # leaving them with things like bootstrap.servers undefined. This checks that bootstrap
+        # servers are defined so we can catch those issues before they cause more problems.
+        virtual do |data, errors|
+          next unless errors.empty?
+
+          kafka = data.fetch(:kafka)
+
+          next if kafka.key?(:'bootstrap.servers')
+
+          [[%w[kafka bootstrap.servers], :missing]]
+        end
+
+        virtual do |data, errors|
+          next unless errors.empty?
+          next unless ::Karafka::App.config.strict_topics_namespacing
+
+          value = data.fetch(:name)
+          namespacing_chars_count = value.chars.find_all { |c| ['.', '_'].include?(c) }.uniq.size
+
+          next if namespacing_chars_count <= 1
+
+          [[%w[name], :inconsistent_namespacing]]
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/active_job/contracts/topic.rb

@@ -14,7 +14,7 @@ module Karafka
                 File.read(
                   File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
                 )
-              ).fetch('en').fetch('validations').fetch('topic')
+              ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
             end
 
             virtual do |data, errors|

data/lib/karafka/routing/features/active_job/topic.rb

@@ -6,6 +6,15 @@ module Karafka
       class ActiveJob < Base
         # Topic extensions to be able to check if given topic is ActiveJob topic
         module Topic
+          # This method calls the parent class initializer and then sets up the
+          # extra instance variable to nil. The explicit initialization
+          # to nil is included as an optimization for Ruby's object shapes system,
+          # which improves memory layout and access performance.
+          def initialize(...)
+            super
+            @active_job = nil
+          end
+
           # @param active [Boolean] should this topic be considered one working with ActiveJob
           #
           # @note Since this feature supports only one setting (active), we can use the old API

data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb

@@ -13,7 +13,7 @@ module Karafka
                 File.read(
                   File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
                 )
-              ).fetch('en').fetch('validations').fetch('topic')
+              ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
             end
 
             nested :dead_letter_queue do

data/lib/karafka/routing/features/dead_letter_queue/topic.rb

@@ -11,6 +11,15 @@ module Karafka
 
           private_constant :DEFAULT_MAX_RETRIES
 
+          # This method calls the parent class initializer and then sets up the
+          # extra instance variable to nil. The explicit initialization
+          # to nil is included as an optimization for Ruby's object shapes system,
+          # which improves memory layout and access performance.
+          def initialize(...)
+            super
+            @dead_letter_queue = nil
+          end
+
           # @param max_retries [Integer] after how many retries should we move data to dlq
           # @param topic [String, false] where the messages should be moved if failing or false
           #   if we do not want to move it anywhere and just skip

data/lib/karafka/routing/features/declaratives/contracts/topic.rb

@@ -13,7 +13,7 @@ module Karafka
                 File.read(
                   File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
                 )
-              ).fetch('en').fetch('validations').fetch('topic')
+              ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
             end
 
             nested :declaratives do

data/lib/karafka/routing/features/declaratives/topic.rb

@@ -6,6 +6,15 @@ module Karafka
       class Declaratives < Base
         # Extension for managing Kafka topic configuration
         module Topic
+          # This method calls the parent class initializer and then sets up the
+          # extra instance variable to nil. The explicit initialization
+          # to nil is included as an optimization for Ruby's object shapes system,
+          # which improves memory layout and access performance.
+          def initialize(...)
+            super
+            @declaratives = nil
+          end
+
           # @param active [Boolean] is the topic structure management feature active
           # @param partitions [Integer]
           # @param replication_factor [Integer]

data/lib/karafka/routing/features/deserializers/contracts/topic.rb

@@ -13,7 +13,7 @@ module Karafka
                 File.read(
                   File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
                 )
-              ).fetch('en').fetch('validations').fetch('topic')
+              ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
             end
 
             nested :deserializers do

data/lib/karafka/routing/features/deserializers/topic.rb

@@ -8,6 +8,15 @@ module Karafka
         # Routing topic deserializers API. It allows to configure deserializers for various
         # components of each message.
         module Topic
+          # This method calls the parent class initializer and then sets up the
+          # extra instance variable to nil. The explicit initialization
+          # to nil is included as an optimization for Ruby's object shapes system,
+          # which improves memory layout and access performance.
+          def initialize(...)
+            super
+            @deserializers = nil
+          end
+
           # Allows for setting all the deserializers with standard defaults
           # @param payload [Object] Deserializer for the message payload
           # @param key [Object] deserializer for the message key

data/lib/karafka/routing/features/eofed/contracts/topic.rb

@@ -13,7 +13,7 @@ module Karafka
                 File.read(
                   File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
                 )
-              ).fetch('en').fetch('validations').fetch('topic')
+              ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
             end
 
             nested :eofed do

data/lib/karafka/routing/features/eofed/topic.rb

@@ -6,6 +6,15 @@ module Karafka
       class Eofed < Base
         # Routing topic eofed API
         module Topic
+          # This method calls the parent class initializer and then sets up the
+          # extra instance variable to nil. The explicit initialization
+          # to nil is included as an optimization for Ruby's object shapes system,
+          # which improves memory layout and access performance.
+          def initialize(...)
+            super
+            @eofed = nil
+          end
+
           # @param active [Boolean] should the `#eofed` job run on eof
           def eofed(active = false)
             @eofed ||= Config.new(

data/lib/karafka/routing/features/inline_insights/contracts/topic.rb

@@ -13,7 +13,7 @@ module Karafka
                 File.read(
                   File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
                 )
-              ).fetch('en').fetch('validations').fetch('topic')
+              ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
             end
 
             nested :inline_insights do

data/lib/karafka/routing/features/inline_insights/topic.rb

@@ -6,6 +6,15 @@ module Karafka
       class InlineInsights < Base
         # Routing topic inline insights API
         module Topic
+          # This method calls the parent class initializer and then sets up the
+          # extra instance variable to nil. The explicit initialization
+          # to nil is included as an optimization for Ruby's object shapes system,
+          # which improves memory layout and access performance.
+          def initialize(...)
+            super
+            @inline_insights = nil
+          end
+
           # @param active [Boolean] should inline insights be activated
           def inline_insights(active = false)
             @inline_insights ||= Config.new(

data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb

@@ -13,7 +13,7 @@ module Karafka
                 File.read(
                   File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
                 )
-              ).fetch('en').fetch('validations').fetch('topic')
+              ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
             end
 
             nested :manual_offset_management do

data/lib/karafka/routing/features/manual_offset_management/topic.rb

@@ -6,6 +6,15 @@ module Karafka
       class ManualOffsetManagement < Base
         # Topic extensions to be able to manage manual offset management settings
         module Topic
+          # This method calls the parent class initializer and then sets up the
+          # extra instance variable to nil. The explicit initialization
+          # to nil is included as an optimization for Ruby's object shapes system,
+          # which improves memory layout and access performance.
+          def initialize(...)
+            super
+            @manual_offset_management = nil
+          end
+
           # @param active [Boolean] should we stop managing the offset in Karafka and make the user
           #   responsible for marking messages as consumed.
           # @return [Config] defined config

data/lib/karafka/routing/subscription_group.rb

@@ -148,16 +148,7 @@ module Karafka
         # end user has configured
         return if kafka.key?(:'client.id')
 
-        # This mitigates an issue for multiplexing and potentially other cases when running
-        # multiple karafka processes on one machine, where librdkafka goes into an infinite
-        # loop when using cooperative-sticky and upscaling.
-        #
-        # @see https://github.com/confluentinc/librdkafka/issues/4783
-        kafka[:'client.id'] = if kafka[:'partition.assignment.strategy'] == 'cooperative-sticky'
-                                "#{client_id}/#{Time.now.to_f}/#{SecureRandom.hex[0..9]}"
-                              else
-                                client_id
-                              end
+        kafka[:'client.id'] = client_id
       end
 
       # If we use static group memberships, there can be a case, where same instance id would

data/lib/karafka/routing/topic.rb

@@ -40,10 +40,18 @@ module Karafka
         #   Karafka 0.6 we can handle multiple Kafka instances with the same process and we can
         #   have same topic name across multiple consumer groups
         @id = "#{consumer_group.id}_#{@name}"
+        @consumer = nil
+        @active_assigned = false
+        @subscription_group_details = nil
+
+        INHERITABLE_ATTRIBUTES.each do |attribute|
+          instance_variable_set("@#{attribute}", nil)
+        end
       end
 
       INHERITABLE_ATTRIBUTES.each do |attribute|
-        attr_writer attribute
+        # Defined below
+        attr_writer attribute unless attribute == :kafka
 
         class_eval <<~RUBY, __FILE__, __LINE__ + 1
           def #{attribute}

data/lib/karafka/server.rb

@@ -3,18 +3,13 @@
 module Karafka
   # Karafka consuming server class
   class Server
-    # How long should we wait on the listeners forceful shutdown when they are stuck beyond the
-    # shutdown timeout before forcing a bypass
-    FORCEFUL_SHUTDOWN_WAIT = 5
-
-    private_constant :FORCEFUL_SHUTDOWN_WAIT
-
     extend Helpers::ConfigImporter.new(
       cli_contract: %i[internal cli contract],
       activity_manager: %i[internal routing activity_manager],
       supervision_sleep: %i[internal supervision_sleep],
       shutdown_timeout: %i[shutdown_timeout],
       forceful_exit_code: %i[internal forceful_exit_code],
+      forceful_shutdown_wait: %i[internal forceful_shutdown_wait],
       process: %i[internal process]
     )
 
@@ -151,7 +146,7 @@ module Karafka
             error: e,
             type: 'app.forceful_stopping.error'
           )
-        end.join(FORCEFUL_SHUTDOWN_WAIT)
+        end.join(forceful_shutdown_wait / 1_000.0)
 
         # We also do not forcefully terminate everything when running in the embedded mode,
         # otherwise we would overwrite the shutdown process of the process that started Karafka

data/lib/karafka/setup/attributes_map.rb

@@ -57,6 +57,8 @@ module Karafka
         group.protocol.type
         group.remote.assignor
         heartbeat.interval.ms
+        https.ca.location
+        https.ca.pem
         interceptors
         internal.termination.signal
         isolation.level
@@ -73,6 +75,7 @@ module Karafka
         message.max.bytes
         metadata.broker.list
         metadata.max.age.ms
+        metadata.recovery.rebootstrap.trigger.ms
         metadata.recovery.strategy
         oauthbearer_token_refresh_cb
         offset.store.method
@@ -100,10 +103,25 @@ module Karafka
         sasl.kerberos.service.name
         sasl.mechanism
         sasl.mechanisms
+        sasl.oauthbearer.assertion.algorithm
+        sasl.oauthbearer.assertion.claim.aud
+        sasl.oauthbearer.assertion.claim.exp.seconds
+        sasl.oauthbearer.assertion.claim.iss
+        sasl.oauthbearer.assertion.claim.jti.include
+        sasl.oauthbearer.assertion.claim.nbf.seconds
+        sasl.oauthbearer.assertion.claim.sub
+        sasl.oauthbearer.assertion.file
+        sasl.oauthbearer.assertion.jwt.template.file
+        sasl.oauthbearer.assertion.private.key.file
+        sasl.oauthbearer.assertion.private.key.passphrase
+        sasl.oauthbearer.assertion.private.key.pem
+        sasl.oauthbearer.client.credentials.client.id
+        sasl.oauthbearer.client.credentials.client.secret
         sasl.oauthbearer.client.id
         sasl.oauthbearer.client.secret
         sasl.oauthbearer.config
         sasl.oauthbearer.extensions
+        sasl.oauthbearer.grant.type
         sasl.oauthbearer.method
         sasl.oauthbearer.scope
         sasl.oauthbearer.token.endpoint.url
@@ -192,6 +210,8 @@ module Karafka
         enable.ssl.certificate.verification
         enabled_events
         error_cb
+        https.ca.location
+        https.ca.pem
         interceptors
         internal.termination.signal
         linger.ms
@@ -208,6 +228,7 @@ module Karafka
         message.timeout.ms
         metadata.broker.list
         metadata.max.age.ms
+        metadata.recovery.rebootstrap.trigger.ms
         metadata.recovery.strategy
         msg_order_cmp
         oauthbearer_token_refresh_cb
@@ -239,10 +260,25 @@ module Karafka
         sasl.kerberos.service.name
         sasl.mechanism
         sasl.mechanisms
+        sasl.oauthbearer.assertion.algorithm
+        sasl.oauthbearer.assertion.claim.aud
+        sasl.oauthbearer.assertion.claim.exp.seconds
+        sasl.oauthbearer.assertion.claim.iss
+        sasl.oauthbearer.assertion.claim.jti.include
+        sasl.oauthbearer.assertion.claim.nbf.seconds
+        sasl.oauthbearer.assertion.claim.sub
+        sasl.oauthbearer.assertion.file
+        sasl.oauthbearer.assertion.jwt.template.file
+        sasl.oauthbearer.assertion.private.key.file
+        sasl.oauthbearer.assertion.private.key.passphrase
+        sasl.oauthbearer.assertion.private.key.pem
+        sasl.oauthbearer.client.credentials.client.id
+        sasl.oauthbearer.client.credentials.client.secret
         sasl.oauthbearer.client.id
         sasl.oauthbearer.client.secret
         sasl.oauthbearer.config
         sasl.oauthbearer.extensions
+        sasl.oauthbearer.grant.type
         sasl.oauthbearer.method
         sasl.oauthbearer.scope
         sasl.oauthbearer.token.endpoint.url

data/lib/karafka/setup/config.rb

@@ -29,8 +29,8 @@ module Karafka
         setting :entity, default: ''
       end
 
-      # option client_id [String] kafka client_id - used to provide
-      #   default Kafka groups namespaces and identify that app in kafka
+      # option client_id [String] kafka client_id - used to uniquely identify given client instance
+      #   Used only for logging.
       setting :client_id, default: 'karafka'
       # option logger [Instance] logger that we want to use
       setting :logger, default: ::Karafka::Instrumentation::Logger.new
@@ -177,16 +177,15 @@ module Karafka
         setting :supervision_sleep, default: 0.1
         # What system exit code should we use when we terminated forcefully
         setting :forceful_exit_code, default: 2
+        # How long should we wait on the listeners forceful shutdown when they are stuck beyond the
+        # shutdown timeout before forcing a bypass
+        setting :forceful_shutdown_wait, default: 5_000
 
         setting :swarm do
           # Manager for swarm nodes control
           setting :manager, default: Swarm::Manager.new
           # Exit code we exit an orphaned child with to indicate something went wrong
           setting :orphaned_exit_code, default: 3
-          # syscall number for https://man7.org/linux/man-pages/man2/pidfd_open.2.html
-          setting :pidfd_open_syscall, default: 434
-          # syscall number for https://man7.org/linux/man-pages/man2/pidfd_send_signal.2.html
-          setting :pidfd_signal_syscall, default: 424
           # How often (in ms) should we control our nodes
           # This is maximum time after which we will check. This can happen more often in case of
           # system events.
@@ -208,7 +207,7 @@ module Karafka
         setting :cli do
           # option contract [Object] cli setup validation contract (in the context of options and
           # topics)
-          setting :contract, default: Contracts::ServerCliOptions.new
+          setting :contract, default: ::Karafka::Cli::Contracts::Server.new
         end
 
         setting :routing do