rdkafka 0.16.0.beta1 → 0.16.0

data/lib/rdkafka/callbacks.rb

@@ -113,6 +113,42 @@ module Rdkafka
       end
     end
 
+    class DescribeConfigsResult
+      attr_reader :result_error, :error_string, :results, :results_count
+
+      def initialize(event_ptr)
+        @results=[]
+        @result_error = Rdkafka::Bindings.rd_kafka_event_error(event_ptr)
+        @error_string = Rdkafka::Bindings.rd_kafka_event_error_string(event_ptr)
+
+        if @result_error == 0
+          configs_describe_result = Rdkafka::Bindings.rd_kafka_event_DescribeConfigs_result(event_ptr)
+          # Get the number of matching acls
+          pointer_to_size_t = FFI::MemoryPointer.new(:int32)
+          @results = Rdkafka::Bindings.rd_kafka_DescribeConfigs_result_resources(configs_describe_result, pointer_to_size_t)
+          @results_count = pointer_to_size_t.read_int
+        end
+      end
+    end
+
+    class IncrementalAlterConfigsResult
+      attr_reader :result_error, :error_string, :results, :results_count
+
+      def initialize(event_ptr)
+        @results=[]
+        @result_error = Rdkafka::Bindings.rd_kafka_event_error(event_ptr)
+        @error_string = Rdkafka::Bindings.rd_kafka_event_error_string(event_ptr)
+
+        if @result_error == 0
+          incremental_alter_result = Rdkafka::Bindings.rd_kafka_event_IncrementalAlterConfigs_result(event_ptr)
+          # Get the number of matching acls
+          pointer_to_size_t = FFI::MemoryPointer.new(:int32)
+          @results = Rdkafka::Bindings.rd_kafka_IncrementalAlterConfigs_result_resources(incremental_alter_result, pointer_to_size_t)
+          @results_count = pointer_to_size_t.read_int
+        end
+      end
+    end
+
     # FFI Function used for Create Topic and Delete Topic callbacks
     BackgroundEventCallbackFunction = FFI::Function.new(
         :void, [:pointer, :pointer, :pointer]
@@ -123,20 +159,24 @@ module Rdkafka
     # @private
     class BackgroundEventCallback
       def self.call(_, event_ptr, _)
-        event_type = Rdkafka::Bindings.rd_kafka_event_type(event_ptr)
-        if event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_CREATETOPICS_RESULT
+        case Rdkafka::Bindings.rd_kafka_event_type(event_ptr)
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATETOPICS_RESULT
           process_create_topic(event_ptr)
-        elsif event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_DELETETOPICS_RESULT
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBECONFIGS_RESULT
+          process_describe_configs(event_ptr)
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_INCREMENTALALTERCONFIGS_RESULT
+          process_incremental_alter_configs(event_ptr)
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETETOPICS_RESULT
           process_delete_topic(event_ptr)
-        elsif event_type == Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS_RESULT
+        when Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS_RESULT
           process_create_partitions(event_ptr)
-        elsif event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_CREATEACLS_RESULT
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATEACLS_RESULT
           process_create_acl(event_ptr)
-        elsif event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEACLS_RESULT
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEACLS_RESULT
           process_delete_acl(event_ptr)
-        elsif event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBEACLS_RESULT
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBEACLS_RESULT
           process_describe_acl(event_ptr)
-        elsif event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEGROUPS_RESULT
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEGROUPS_RESULT
           process_delete_groups(event_ptr)
         end
       end
@@ -161,6 +201,42 @@ module Rdkafka
         end
       end
 
+      def self.process_describe_configs(event_ptr)
+        describe_configs = DescribeConfigsResult.new(event_ptr)
+        describe_configs_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+        if describe_configs_handle = Rdkafka::Admin::DescribeConfigsHandle.remove(describe_configs_handle_ptr.address)
+          describe_configs_handle[:response] = describe_configs.result_error
+          describe_configs_handle[:response_string] = describe_configs.error_string
+          describe_configs_handle[:pending] = false
+
+          if describe_configs.result_error == 0
+            describe_configs_handle[:config_entries] = describe_configs.results
+            describe_configs_handle[:entry_count] = describe_configs.results_count
+          end
+
+          describe_configs_handle.unlock
+        end
+      end
+
+      def self.process_incremental_alter_configs(event_ptr)
+        incremental_alter = IncrementalAlterConfigsResult.new(event_ptr)
+        incremental_alter_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+        if incremental_alter_handle = Rdkafka::Admin::IncrementalAlterConfigsHandle.remove(incremental_alter_handle_ptr.address)
+          incremental_alter_handle[:response] = incremental_alter.result_error
+          incremental_alter_handle[:response_string] = incremental_alter.error_string
+          incremental_alter_handle[:pending] = false
+
+          if incremental_alter.result_error == 0
+            incremental_alter_handle[:config_entries] = incremental_alter.results
+            incremental_alter_handle[:entry_count] = incremental_alter.results_count
+          end
+
+          incremental_alter_handle.unlock
+        end
+      end
+
       def self.process_delete_groups(event_ptr)
         delete_groups_result = Rdkafka::Bindings.rd_kafka_event_DeleteGroups_result(event_ptr)
 
@@ -263,7 +339,7 @@ module Rdkafka
           describe_acl_handle[:response_string] = describe_acl.error_string
 
           if describe_acl.result_error == 0
-            describe_acl_handle[:acls]       = describe_acl.matching_acls
+            describe_acl_handle[:acls] = describe_acl.matching_acls
             describe_acl_handle[:acls_count] = describe_acl.matching_acls_count
           end
 

data/lib/rdkafka/consumer/headers.rb

@@ -4,17 +4,7 @@ module Rdkafka
   class Consumer
     # Interface to return headers for a consumer message
     module Headers
-      class HashWithSymbolKeysTreatedLikeStrings < Hash
-        def [](key)
-          if key.is_a?(Symbol)
-            Kernel.warn("rdkafka deprecation warning: header access with Symbol key #{key.inspect} treated as a String. " \
-                        "Please change your code to use String keys to avoid this warning. Symbol keys will break in version 1.")
-            super(key.to_s)
-          else
-            super
-          end
-        end
-      end
+      EMPTY_HEADERS = {}.freeze
 
       # Reads a librdkafka native message's headers and returns them as a Ruby Hash
       #
@@ -28,7 +18,7 @@ module Rdkafka
         err = Rdkafka::Bindings.rd_kafka_message_headers(native_message, headers_ptrptr)
 
         if err == Rdkafka::Bindings::RD_KAFKA_RESP_ERR__NOENT
-          return {}
+          return EMPTY_HEADERS
         elsif err != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
           raise Rdkafka::RdkafkaError.new(err, "Error reading message headers")
         end
@@ -39,7 +29,7 @@ module Rdkafka
         value_ptrptr = FFI::MemoryPointer.new(:pointer)
         size_ptr = Rdkafka::Bindings::SizePtr.new
 
-        headers = HashWithSymbolKeysTreatedLikeStrings.new
+        headers = {}
 
         idx = 0
         loop do

data/lib/rdkafka/producer/delivery_handle.rb

@@ -14,6 +14,10 @@ module Rdkafka
       # @return [Object, nil] label set during message production or nil by default
       attr_accessor :label
 
+      # @return [String] topic where we are trying to send the message
+      # We use this instead of reading from `topic_name` pointer to save on memory allocations
+      attr_accessor :topic
+
       # @return [String] the name of the operation (e.g. "delivery")
       def operation_name
         "delivery"
@@ -26,7 +30,7 @@ module Rdkafka
           self[:offset],
           # For part of errors, we will not get a topic name reference and in cases like this
           # we should not return it
-          self[:topic_name].null? ? nil : self[:topic_name].read_string,
+          topic,
           self[:response] != 0 ? RdkafkaError.new(self[:response]) : nil,
           label
         )

data/lib/rdkafka/producer.rb

@@ -9,7 +9,15 @@ module Rdkafka
     # Cache partitions count for 30 seconds
     PARTITIONS_COUNT_TTL = 30
 
-    private_constant :PARTITIONS_COUNT_TTL
+    # Empty hash used as a default
+    EMPTY_HASH = {}.freeze
+
+    private_constant :PARTITIONS_COUNT_TTL, :EMPTY_HASH
+
+    # Raised when there was a critical issue when invoking rd_kafka_topic_new
+    # This is a temporary solution until https://github.com/karafka/rdkafka-ruby/issues/451 is
+    # resolved and this is normalized in all the places
+    class TopicHandleCreationError < RuntimeError; end
 
     # @private
     # Returns the current delivery callback, by default this is nil.
@@ -28,6 +36,8 @@ module Rdkafka
     # @param partitioner_name [String, nil] name of the partitioner we want to use or nil to use
     #   the "consistent_random" default
     def initialize(native_kafka, partitioner_name)
+      @topics_refs_map = {}
+      @topics_configs = {}
       @native_kafka = native_kafka
       @partitioner_name = partitioner_name || "consistent_random"
 
@@ -54,6 +64,52 @@ module Rdkafka
       end
     end
 
+    # Sets alternative set of configuration details that can be set per topic
+    # @note It is not allowed to re-set the same topic config twice because of the underlying
+    #   librdkafka caching
+    # @param topic [String] The topic name
+    # @param config [Hash] config we want to use per topic basis
+    # @param config_hash [Integer] hash of the config. We expect it here instead of computing it,
+    #   because it is already computed during the retrieval attempt in the `#produce` flow.
+    def set_topic_config(topic, config, config_hash)
+      # Ensure lock on topic reference just in case
+      @native_kafka.with_inner do |inner|
+        @topics_refs_map[topic] ||= {}
+        @topics_configs[topic] ||= {}
+
+        return if @topics_configs[topic].key?(config_hash)
+
+        # If config is empty, we create an empty reference that will be used with defaults
+        rd_topic_config = if config.empty?
+                            nil
+                          else
+                            Rdkafka::Bindings.rd_kafka_topic_conf_new.tap do |topic_config|
+                              config.each do |key, value|
+                                error_buffer = FFI::MemoryPointer.new(:char, 256)
+                                result = Rdkafka::Bindings.rd_kafka_topic_conf_set(
+                                  topic_config,
+                                  key.to_s,
+                                  value.to_s,
+                                  error_buffer,
+                                  256
+                                )
+
+                                unless result == :config_ok
+                                  raise Config::ConfigError.new(error_buffer.read_string)
+                                end
+                              end
+                            end
+                          end
+
+        topic_handle = Bindings.rd_kafka_topic_new(inner, topic, rd_topic_config)
+
+        raise TopicHandleCreationError.new("Error creating topic handle for topic #{topic}") if topic_handle.null?
+
+        @topics_configs[topic][config_hash] = config
+        @topics_refs_map[topic][config_hash] = topic_handle
+      end
+    end
+
     # Starts the native Kafka polling thread and kicks off the init polling
     # @note Not needed to run unless explicit start was disabled
     def start
@@ -83,7 +139,18 @@ module Rdkafka
     def close
       return if closed?
       ObjectSpace.undefine_finalizer(self)
-      @native_kafka.close
+
+      @native_kafka.close do
+        # We need to remove the topics references objects before we destroy the producer,
+        # otherwise they would leak out
+        @topics_refs_map.each_value do |refs|
+          refs.each_value do |ref|
+            Rdkafka::Bindings.rd_kafka_topic_destroy(ref)
+          end
+        end
+      end
+
+      @topics_refs_map.clear
     end
 
     # Whether this producer has closed
@@ -182,11 +249,22 @@ module Rdkafka
     # @param timestamp [Time,Integer,nil] Optional timestamp of this message. Integer timestamp is in milliseconds since Jan 1 1970.
     # @param headers [Hash<String,String>] Optional message headers
     # @param label [Object, nil] a label that can be assigned when producing a message that will be part of the delivery handle and the delivery report
+    # @param topic_config [Hash] topic config for given message dispatch. Allows to send messages to topics with different configuration
     #
     # @return [DeliveryHandle] Delivery handle that can be used to wait for the result of producing this message
     #
     # @raise [RdkafkaError] When adding the message to rdkafka's queue failed
-    def produce(topic:, payload: nil, key: nil, partition: nil, partition_key: nil, timestamp: nil, headers: nil, label: nil)
+    def produce(
+      topic:,
+      payload: nil,
+      key: nil,
+      partition: nil,
+      partition_key: nil,
+      timestamp: nil,
+      headers: nil,
+      label: nil,
+      topic_config: EMPTY_HASH
+    )
       closed_producer_check(__method__)
 
       # Start by checking and converting the input
@@ -205,8 +283,20 @@ module Rdkafka
                    key.bytesize
                  end
 
+      topic_config_hash = topic_config.hash
+
+      # Checks if we have the rdkafka topic reference object ready. It saves us on object
+      # allocation and allows to use custom config on demand.
+      set_topic_config(topic, topic_config, topic_config_hash) unless @topics_refs_map.dig(topic, topic_config_hash)
+      topic_ref = @topics_refs_map.dig(topic, topic_config_hash)
+
       if partition_key
         partition_count = partition_count(topic)
+
+        # Check if there are no overrides for the partitioner and use the default one only when
+        # no per-topic is present.
+        partitioner_name = @topics_configs.dig(topic, topic_config_hash, :partitioner) || @partitioner_name
+
         # If the topic is not present, set to -1
         partition = Rdkafka::Bindings.partitioner(partition_key, partition_count, @partitioner_name) if partition_count.positive?
       end
@@ -229,6 +319,7 @@ module Rdkafka
 
       delivery_handle = DeliveryHandle.new
       delivery_handle.label = label
+      delivery_handle.topic = topic
       delivery_handle[:pending] = true
       delivery_handle[:response] = -1
       delivery_handle[:partition] = -1
@@ -236,7 +327,7 @@ module Rdkafka
       DeliveryHandle.register(delivery_handle)
 
       args = [
-        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_TOPIC, :string, topic,
+        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_RKT, :pointer, topic_ref,
         :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_MSGFLAGS, :int, Rdkafka::Bindings::RD_KAFKA_MSG_F_COPY,
         :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_VALUE, :buffer_in, payload, :size_t, payload_size,
         :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_KEY, :buffer_in, key, :size_t, key_size,
@@ -252,7 +343,7 @@ module Rdkafka
           args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_HEADER
           args << :string << key
           args << :pointer << value
-          args << :size_t << value.bytes.size
+          args << :size_t << value.bytesize
         end
       end
 

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.16.0.beta1"
+  VERSION = "0.16.0"
   LIBRDKAFKA_VERSION = "2.3.0"
   LIBRDKAFKA_SOURCE_SHA256 = "2d49c35c77eeb3d42fa61c43757fcbb6a206daa560247154e60642bcdcc14d12"
 end

data/lib/rdkafka.rb

@@ -24,7 +24,13 @@ require "rdkafka/admin/delete_acl_handle"
 require "rdkafka/admin/delete_acl_report"
 require "rdkafka/admin/describe_acl_handle"
 require "rdkafka/admin/describe_acl_report"
+require "rdkafka/admin/describe_configs_handle"
+require "rdkafka/admin/describe_configs_report"
+require "rdkafka/admin/incremental_alter_configs_handle"
+require "rdkafka/admin/incremental_alter_configs_report"
 require "rdkafka/admin/acl_binding_result"
+require "rdkafka/admin/config_binding_result"
+require "rdkafka/admin/config_resource_binding_result"
 require "rdkafka/bindings"
 require "rdkafka/callbacks"
 require "rdkafka/config"

data/rdkafka.gemspec

@@ -15,7 +15,7 @@ Gem::Specification.new do |gem|
   gem.name = 'rdkafka'
   gem.require_paths = ['lib']
   gem.version = Rdkafka::VERSION
-  gem.required_ruby_version = '>= 2.7'
+  gem.required_ruby_version = '>= 3.0'
   gem.extensions = %w(ext/Rakefile)
   gem.cert_chain = %w[certs/cert_chain.pem]