rdkafka 0.28.0 → 0.29.0

data/lib/rdkafka/callbacks/incremental_alter_configs_handler.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Callbacks
+    # Handles `RD_KAFKA_EVENT_INCREMENTALALTERCONFIGS_RESULT` events
+    # @private
+    class IncrementalAlterConfigsHandler < BaseHandler
+      class << self
+        # Resolves the incremental-alter-configs handle from its result event
+        # @param event_ptr [FFI::Pointer] pointer to the event
+        # @return [void]
+        def call(event_ptr)
+          incremental_alter = IncrementalAlterConfigsResult.new(event_ptr)
+          incremental_alter_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+          if incremental_alter_handle = Rdkafka::Admin::IncrementalAlterConfigsHandle.remove(incremental_alter_handle_ptr.address)
+            incremental_alter_handle[:response] = incremental_alter.result_error
+            incremental_alter_handle.broker_message = read_event_string(incremental_alter.error_string)
+
+            # Parsing can raise on per-resource errors. The exception is captured and re-raised on
+            # the waiting thread, since an exception leaving this callback would unwind through
+            # librdkafka native frames.
+            incremental_alter_handle.result = begin
+              if incremental_alter.result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+                Rdkafka::Admin::IncrementalAlterConfigsReport.new(
+                  config_entries: incremental_alter.results,
+                  entry_count: incremental_alter.results_count
+                )
+              else
+                Rdkafka::Admin::IncrementalAlterConfigsReport.new(config_entries: FFI::Pointer::NULL, entry_count: 0)
+              end
+            rescue => e
+              e
+            end
+
+            incremental_alter_handle.unlock
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/callbacks/list_offsets_handler.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Callbacks
+    # Handles `RD_KAFKA_EVENT_LISTOFFSETS_RESULT` events
+    # @private
+    class ListOffsetsHandler < BaseHandler
+      class << self
+        # Resolves the list-offsets handle from its result event
+        # @param event_ptr [FFI::Pointer] pointer to the event
+        # @return [void]
+        def call(event_ptr)
+          list_offsets = ListOffsetsResult.new(event_ptr)
+          list_offsets_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+          if list_offsets_handle = Rdkafka::Admin::ListOffsetsHandle.remove(list_offsets_handle_ptr.address)
+            list_offsets_handle[:response] = list_offsets.result_error
+            list_offsets_handle.broker_message = read_event_string(list_offsets.error_string)
+
+            # Parsing can raise on per-partition errors. The exception is captured and re-raised
+            # on the waiting thread, since an exception leaving this callback would unwind through
+            # librdkafka native frames.
+            list_offsets_handle.result = begin
+              if list_offsets.result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+                Rdkafka::Admin::ListOffsetsReport.new(
+                  result_infos: list_offsets.result_infos,
+                  result_count: list_offsets.result_count
+                )
+              else
+                Rdkafka::Admin::ListOffsetsReport.new(result_infos: FFI::Pointer::NULL, result_count: 0)
+              end
+            rescue => e
+              e
+            end
+
+            list_offsets_handle.unlock
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/callbacks.rb

@@ -98,7 +98,7 @@ module Rdkafka
         @error_string = Rdkafka::Bindings.rd_kafka_error_string(rd_kafka_error_pointer)
         if @result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
           # Get the number of matching acls
-          pointer_to_size_t = FFI::MemoryPointer.new(:int32)
+          pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
           @matching_acls = Rdkafka::Bindings.rd_kafka_DeleteAcls_result_response_matching_acls(acl_result_pointer, pointer_to_size_t)
           @matching_acls_count = pointer_to_size_t.read_int
         end
@@ -129,7 +129,7 @@ module Rdkafka
         if @result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
           acl_describe_result = Rdkafka::Bindings.rd_kafka_event_DescribeAcls_result(event_ptr)
           # Get the number of matching acls
-          pointer_to_size_t = FFI::MemoryPointer.new(:int32)
+          pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
           @matching_acls = Rdkafka::Bindings.rd_kafka_DescribeAcls_result_acls(acl_describe_result, pointer_to_size_t)
           @matching_acls_count = pointer_to_size_t.read_int
         end
@@ -151,7 +151,7 @@ module Rdkafka
         if @result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
           configs_describe_result = Rdkafka::Bindings.rd_kafka_event_DescribeConfigs_result(event_ptr)
           # Get the number of matching acls
-          pointer_to_size_t = FFI::MemoryPointer.new(:int32)
+          pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
           @results = Rdkafka::Bindings.rd_kafka_DescribeConfigs_result_resources(configs_describe_result, pointer_to_size_t)
           @results_count = pointer_to_size_t.read_int
         end
@@ -173,7 +173,7 @@ module Rdkafka
         if @result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
           incremental_alter_result = Rdkafka::Bindings.rd_kafka_event_IncrementalAlterConfigs_result(event_ptr)
           # Get the number of matching acls
-          pointer_to_size_t = FFI::MemoryPointer.new(:int32)
+          pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
           @results = Rdkafka::Bindings.rd_kafka_IncrementalAlterConfigs_result_resources(incremental_alter_result, pointer_to_size_t)
           @results_count = pointer_to_size_t.read_int
         end
@@ -208,232 +208,26 @@ module Rdkafka
       # @param event_ptr [FFI::Pointer] pointer to the event
       # @param _opaque_ptr [FFI::Pointer] unused opaque pointer
       def self.call(_client_ptr, event_ptr, _opaque_ptr)
-        case Rdkafka::Bindings.rd_kafka_event_type(event_ptr)
-        when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATETOPICS_RESULT
-          process_create_topic(event_ptr)
-        when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBECONFIGS_RESULT
-          process_describe_configs(event_ptr)
-        when Rdkafka::Bindings::RD_KAFKA_EVENT_INCREMENTALALTERCONFIGS_RESULT
-          process_incremental_alter_configs(event_ptr)
-        when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETETOPICS_RESULT
-          process_delete_topic(event_ptr)
-        when Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS_RESULT
-          process_create_partitions(event_ptr)
-        when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATEACLS_RESULT
-          process_create_acl(event_ptr)
-        when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEACLS_RESULT
-          process_delete_acl(event_ptr)
-        when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBEACLS_RESULT
-          process_describe_acl(event_ptr)
-        when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEGROUPS_RESULT
-          process_delete_groups(event_ptr)
-        when Rdkafka::Bindings::RD_KAFKA_EVENT_LISTOFFSETS_RESULT
-          process_list_offsets(event_ptr)
-        end
-      end
-
-      private
-
-      # Processes create topic result event
-      # @param event_ptr [FFI::Pointer] pointer to the event
-      def self.process_create_topic(event_ptr)
-        create_topics_result = Rdkafka::Bindings.rd_kafka_event_CreateTopics_result(event_ptr)
-
-        # Get the number of create topic results
-        pointer_to_size_t = FFI::MemoryPointer.new(:int32)
-        create_topic_result_array = Rdkafka::Bindings.rd_kafka_CreateTopics_result_topics(create_topics_result, pointer_to_size_t)
-        create_topic_results = TopicResult.create_topic_results_from_array(pointer_to_size_t.read_int, create_topic_result_array)
-        create_topic_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
-
-        if create_topic_handle = Rdkafka::Admin::CreateTopicHandle.remove(create_topic_handle_ptr.address)
-          create_topic_handle[:response] = create_topic_results[0].result_error
-          create_topic_handle[:error_string] = create_topic_results[0].error_string
-          create_topic_handle[:result_name] = create_topic_results[0].result_name
-
-          create_topic_handle.unlock
-        end
-      end
-
-      # Processes describe configs result event
-      # @param event_ptr [FFI::Pointer] pointer to the event
-      def self.process_describe_configs(event_ptr)
-        describe_configs = DescribeConfigsResult.new(event_ptr)
-        describe_configs_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
-
-        if describe_configs_handle = Rdkafka::Admin::DescribeConfigsHandle.remove(describe_configs_handle_ptr.address)
-          describe_configs_handle[:response] = describe_configs.result_error
-          describe_configs_handle[:response_string] = describe_configs.error_string
-          describe_configs_handle[:pending] = false
-
-          if describe_configs.result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
-            describe_configs_handle[:config_entries] = describe_configs.results
-            describe_configs_handle[:entry_count] = describe_configs.results_count
-          end
-
-          describe_configs_handle.unlock
-        end
-      end
-
-      # Processes incremental alter configs result event
-      # @param event_ptr [FFI::Pointer] pointer to the event
-      def self.process_incremental_alter_configs(event_ptr)
-        incremental_alter = IncrementalAlterConfigsResult.new(event_ptr)
-        incremental_alter_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
-
-        if incremental_alter_handle = Rdkafka::Admin::IncrementalAlterConfigsHandle.remove(incremental_alter_handle_ptr.address)
-          incremental_alter_handle[:response] = incremental_alter.result_error
-          incremental_alter_handle[:response_string] = incremental_alter.error_string
-          incremental_alter_handle[:pending] = false
-
-          if incremental_alter.result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
-            incremental_alter_handle[:config_entries] = incremental_alter.results
-            incremental_alter_handle[:entry_count] = incremental_alter.results_count
-          end
-
-          incremental_alter_handle.unlock
-        end
-      end
-
-      # Processes delete groups result event
-      # @param event_ptr [FFI::Pointer] pointer to the event
-      def self.process_delete_groups(event_ptr)
-        delete_groups_result = Rdkafka::Bindings.rd_kafka_event_DeleteGroups_result(event_ptr)
-
-        # Get the number of delete group results
-        pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
-        delete_group_result_array = Rdkafka::Bindings.rd_kafka_DeleteGroups_result_groups(delete_groups_result, pointer_to_size_t)
-        delete_group_results = GroupResult.create_group_results_from_array(pointer_to_size_t.read_int, delete_group_result_array) # TODO fix this
-        delete_group_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
-
-        if (delete_group_handle = Rdkafka::Admin::DeleteGroupsHandle.remove(delete_group_handle_ptr.address))
-          delete_group_handle[:response] = delete_group_results[0].result_error
-          delete_group_handle[:error_string] = delete_group_results[0].error_string
-          delete_group_handle[:result_name] = delete_group_results[0].result_name
-
-          delete_group_handle.unlock
-        end
-      end
-
-      # Processes delete topic result event
-      # @param event_ptr [FFI::Pointer] pointer to the event
-      def self.process_delete_topic(event_ptr)
-        delete_topics_result = Rdkafka::Bindings.rd_kafka_event_DeleteTopics_result(event_ptr)
-
-        # Get the number of topic results
-        pointer_to_size_t = FFI::MemoryPointer.new(:int32)
-        delete_topic_result_array = Rdkafka::Bindings.rd_kafka_DeleteTopics_result_topics(delete_topics_result, pointer_to_size_t)
-        delete_topic_results = TopicResult.create_topic_results_from_array(pointer_to_size_t.read_int, delete_topic_result_array)
-        delete_topic_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
-
-        if delete_topic_handle = Rdkafka::Admin::DeleteTopicHandle.remove(delete_topic_handle_ptr.address)
-          delete_topic_handle[:response] = delete_topic_results[0].result_error
-          delete_topic_handle[:error_string] = delete_topic_results[0].error_string
-          delete_topic_handle[:result_name] = delete_topic_results[0].result_name
-
-          delete_topic_handle.unlock
-        end
-      end
-
-      # Processes create partitions result event
-      # @param event_ptr [FFI::Pointer] pointer to the event
-      def self.process_create_partitions(event_ptr)
-        create_partitionss_result = Rdkafka::Bindings.rd_kafka_event_CreatePartitions_result(event_ptr)
-
-        # Get the number of create topic results
-        pointer_to_size_t = FFI::MemoryPointer.new(:int32)
-        create_partitions_result_array = Rdkafka::Bindings.rd_kafka_CreatePartitions_result_topics(create_partitionss_result, pointer_to_size_t)
-        create_partitions_results = TopicResult.create_topic_results_from_array(pointer_to_size_t.read_int, create_partitions_result_array)
-        create_partitions_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
-
-        if create_partitions_handle = Rdkafka::Admin::CreatePartitionsHandle.remove(create_partitions_handle_ptr.address)
-          create_partitions_handle[:response] = create_partitions_results[0].result_error
-          create_partitions_handle[:error_string] = create_partitions_results[0].error_string
-          create_partitions_handle[:result_name] = create_partitions_results[0].result_name
-
-          create_partitions_handle.unlock
-        end
-      end
-
-      # Processes create ACL result event
-      # @param event_ptr [FFI::Pointer] pointer to the event
-      def self.process_create_acl(event_ptr)
-        create_acls_result = Rdkafka::Bindings.rd_kafka_event_CreateAcls_result(event_ptr)
-
-        # Get the number of acl results
-        pointer_to_size_t = FFI::MemoryPointer.new(:int32)
-        create_acl_result_array = Rdkafka::Bindings.rd_kafka_CreateAcls_result_acls(create_acls_result, pointer_to_size_t)
-        create_acl_results = CreateAclResult.create_acl_results_from_array(pointer_to_size_t.read_int, create_acl_result_array)
-        create_acl_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
-
-        if create_acl_handle = Rdkafka::Admin::CreateAclHandle.remove(create_acl_handle_ptr.address)
-          create_acl_handle[:response] = create_acl_results[0].result_error
-          create_acl_handle[:response_string] = create_acl_results[0].error_string
-
-          create_acl_handle.unlock
-        end
-      end
-
-      # Processes delete ACL result event
-      # @param event_ptr [FFI::Pointer] pointer to the event
-      def self.process_delete_acl(event_ptr)
-        delete_acls_result = Rdkafka::Bindings.rd_kafka_event_DeleteAcls_result(event_ptr)
-
-        # Get the number of acl results
-        pointer_to_size_t = FFI::MemoryPointer.new(:int32)
-        delete_acl_result_responses = Rdkafka::Bindings.rd_kafka_DeleteAcls_result_responses(delete_acls_result, pointer_to_size_t)
-        delete_acl_results = DeleteAclResult.delete_acl_results_from_array(pointer_to_size_t.read_int, delete_acl_result_responses)
-        delete_acl_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
-
-        if delete_acl_handle = Rdkafka::Admin::DeleteAclHandle.remove(delete_acl_handle_ptr.address)
-          delete_acl_handle[:response] = delete_acl_results[0].result_error
-          delete_acl_handle[:response_string] = delete_acl_results[0].error_string
-
-          if delete_acl_results[0].result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
-            delete_acl_handle[:matching_acls] = delete_acl_results[0].matching_acls
-            delete_acl_handle[:matching_acls_count] = delete_acl_results[0].matching_acls_count
-          end
-
-          delete_acl_handle.unlock
-        end
-      end
-
-      # Processes describe ACL result event
-      # @param event_ptr [FFI::Pointer] pointer to the event
-      def self.process_describe_acl(event_ptr)
-        describe_acl = DescribeAclResult.new(event_ptr)
-        describe_acl_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
-
-        if describe_acl_handle = Rdkafka::Admin::DescribeAclHandle.remove(describe_acl_handle_ptr.address)
-          describe_acl_handle[:response] = describe_acl.result_error
-          describe_acl_handle[:response_string] = describe_acl.error_string
-
-          if describe_acl.result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
-            describe_acl_handle[:acls] = describe_acl.matching_acls
-            describe_acl_handle[:acls_count] = describe_acl.matching_acls_count
-          end
-
-          describe_acl_handle.unlock
+        handler = case Rdkafka::Bindings.rd_kafka_event_type(event_ptr)
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATETOPICS_RESULT then CreateTopicHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETETOPICS_RESULT then DeleteTopicHandler
+        when Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS_RESULT then CreatePartitionsHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEGROUPS_RESULT then DeleteGroupsHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATEACLS_RESULT then CreateAclHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEACLS_RESULT then DeleteAclHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBEACLS_RESULT then DescribeAclHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBECONFIGS_RESULT then DescribeConfigsHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_INCREMENTALALTERCONFIGS_RESULT then IncrementalAlterConfigsHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_LISTOFFSETS_RESULT then ListOffsetsHandler
         end
-      end
 
-      # Processes list offsets result event
-      # @param event_ptr [FFI::Pointer] pointer to the event
-      def self.process_list_offsets(event_ptr)
-        list_offsets = ListOffsetsResult.new(event_ptr)
-        list_offsets_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
-
-        if list_offsets_handle = Rdkafka::Admin::ListOffsetsHandle.remove(list_offsets_handle_ptr.address)
-          list_offsets_handle[:response] = list_offsets.result_error
-          list_offsets_handle[:response_string] = list_offsets.error_string
-          list_offsets_handle[:pending] = false
-
-          if list_offsets.result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
-            list_offsets_handle[:result_infos] = list_offsets.result_infos
-            list_offsets_handle[:result_count] = list_offsets.result_count
-          end
-
-          list_offsets_handle.unlock
-        end
+        handler&.call(event_ptr)
+      ensure
+        # Background queue events are owned by the application and must be destroyed once
+        # served (see rd_kafka_conf_set_background_event_cb). All event-owned memory (result
+        # arrays, strings) has to be copied into Ruby objects by the handlers above before this
+        # runs - nothing may retain pointers into the event past this point.
+        Rdkafka::Bindings.rd_kafka_event_destroy(event_ptr)
       end
     end
 
@@ -453,23 +247,35 @@ module Rdkafka
           delivery_handle[:response] = message[:err]
           delivery_handle[:partition] = message[:partition]
           delivery_handle[:offset] = message[:offset]
-          delivery_handle[:topic_name] = FFI::MemoryPointer.from_string(topic_name)
-
-          # Call delivery callback on opaque
-          if opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
-            opaque.call_delivery_callback(
-              Rdkafka::Producer::DeliveryReport.new(
-                message[:partition],
-                message[:offset],
-                topic_name,
-                message[:err],
-                delivery_handle.label
-              ),
-              delivery_handle
-            )
+          # The topic is not stored on the handle here: it is already set as the handle's
+          # `topic` Ruby attribute during produce, which spares a native string copy that
+          # would otherwise be allocated and retained for every message.
+
+          begin
+            # Call delivery callback on opaque
+            if opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
+              opaque.call_delivery_callback(
+                Rdkafka::Producer::DeliveryReport.new(
+                  message[:partition],
+                  message[:offset],
+                  topic_name,
+                  message[:err],
+                  delivery_handle.label
+                ),
+                delivery_handle
+              )
+            end
+          rescue Exception => err
+            # A user delivery callback that raises must not escape this FFI callback: it runs on
+            # librdkafka's polling thread (abort_on_exception = true), so an escaping exception
+            # would crash the whole process. We log and swallow it, matching the rebalance
+            # callback. The `ensure` below still unlocks the handle, so `wait` returns the
+            # delivery report instead of blocking until its timeout and raising
+            # WaitTimeoutError for a message that was in fact delivered.
+            Rdkafka::Config.logger.error("Unhandled exception: #{err.class} - #{err.message}")
+          ensure
+            delivery_handle.unlock
           end
-
-          delivery_handle.unlock
         end
       end
     end

data/lib/rdkafka/config.rb

@@ -342,47 +342,55 @@ module Rdkafka
     # @param opaque [Object, nil] optional opaque pointer for callbacks
     # @return [FFI::Pointer] native rdkafka configuration pointer
     def native_config(opaque = nil)
-      Rdkafka::Bindings.rd_kafka_conf_new.tap do |config|
-        # Create config
-        @config_hash.merge(REQUIRED_CONFIG).each do |key, value|
-          error_buffer = FFI::MemoryPointer.from_string(" " * 256)
-          result = Rdkafka::Bindings.rd_kafka_conf_set(
-            config,
-            key.to_s,
-            value.to_s,
-            error_buffer,
-            256
-          )
-          unless result == :config_ok
-            raise ConfigError.new(error_buffer.read_string)
-          end
+      config = Rdkafka::Bindings.rd_kafka_conf_new
+
+      # Create config
+      @config_hash.merge(REQUIRED_CONFIG).each do |key, value|
+        error_buffer = FFI::MemoryPointer.from_string(" " * 256)
+        result = Rdkafka::Bindings.rd_kafka_conf_set(
+          config,
+          key.to_s,
+          value.to_s,
+          error_buffer,
+          256
+        )
+        unless result == :config_ok
+          raise ConfigError.new(error_buffer.read_string)
         end
+      end
 
-        # Set opaque pointer that's used as a proxy for callbacks
-        if opaque
-          pointer = ::FFI::Pointer.new(:pointer, opaque.object_id)
-          Rdkafka::Bindings.rd_kafka_conf_set_opaque(config, pointer)
+      # Set opaque pointer that's used as a proxy for callbacks
+      if opaque
+        pointer = ::FFI::Pointer.new(:pointer, opaque.object_id)
+        Rdkafka::Bindings.rd_kafka_conf_set_opaque(config, pointer)
 
-          # Store opaque with the pointer as key. We use this approach instead
-          # of trying to convert the pointer to a Ruby object because there is
-          # no risk of a segfault this way.
-          Rdkafka::Config.opaques[pointer.to_i] = opaque
-        end
+        # Store opaque with the pointer as key. We use this approach instead
+        # of trying to convert the pointer to a Ruby object because there is
+        # no risk of a segfault this way.
+        Rdkafka::Config.opaques[pointer.to_i] = opaque
+      end
 
-        # Set log callback
-        Rdkafka::Bindings.rd_kafka_conf_set_log_cb(config, Rdkafka::Bindings::LogCallback)
+      # Set log callback
+      Rdkafka::Bindings.rd_kafka_conf_set_log_cb(config, Rdkafka::Bindings::LogCallback)
 
-        # Set stats callback
-        Rdkafka::Bindings.rd_kafka_conf_set_stats_cb(config, Rdkafka::Bindings::StatsCallback)
+      # Set stats callback
+      Rdkafka::Bindings.rd_kafka_conf_set_stats_cb(config, Rdkafka::Bindings::StatsCallback)
 
-        # Set error callback
-        Rdkafka::Bindings.rd_kafka_conf_set_error_cb(config, Rdkafka::Bindings::ErrorCallback)
+      # Set error callback
+      Rdkafka::Bindings.rd_kafka_conf_set_error_cb(config, Rdkafka::Bindings::ErrorCallback)
 
-        # Set oauth callback
-        if Rdkafka::Config.oauthbearer_token_refresh_callback
-          Rdkafka::Bindings.rd_kafka_conf_set_oauthbearer_token_refresh_cb(config, Rdkafka::Bindings::OAuthbearerTokenRefreshCallback)
-        end
+      # Set oauth callback
+      if Rdkafka::Config.oauthbearer_token_refresh_callback
+        Rdkafka::Bindings.rd_kafka_conf_set_oauthbearer_token_refresh_cb(config, Rdkafka::Bindings::OAuthbearerTokenRefreshCallback)
       end
+
+      config
+    rescue Exception
+      # rd_kafka_conf_new allocates a conf the caller owns until it is handed to rd_kafka_new. If
+      # we raise before returning it (e.g. an invalid option in the loop above), destroy it here
+      # so the native config is not leaked.
+      Rdkafka::Bindings.rd_kafka_conf_destroy(config) if config
+      raise
     end
 
     # Creates a native Kafka handle
@@ -400,6 +408,10 @@ module Rdkafka
       )
 
       if handle.null?
+        # On success rd_kafka_new takes ownership of the conf and frees it itself; on failure
+        # librdkafka keeps application ownership (see rdkafka.c), so we must destroy it here to
+        # avoid leaking the conf on every failed client creation.
+        Rdkafka::Bindings.rd_kafka_conf_destroy(config)
         raise ClientCreationError.new(error_buffer.read_string)
       end
 

data/lib/rdkafka/consumer/headers.rb

@@ -7,6 +7,11 @@ module Rdkafka
       # Empty frozen hash used when there are no headers
       EMPTY_HEADERS = {}.freeze
 
+      # Key under which reusable scratch pointers are stored on the current thread/fiber
+      SCRATCH_KEY = :rdkafka_headers_scratch
+
+      private_constant :SCRATCH_KEY
+
       # Reads a librdkafka native message's headers and returns them as a Ruby Hash
       # where each key maps to either a String (single value) or Array<String> (multiple values)
       # to support duplicate headers per KIP-82
@@ -17,7 +22,20 @@ module Rdkafka
       # @return [Hash{String => String, Array<String>}] headers Hash for the native_message
       # @raise [Rdkafka::RdkafkaError] when fail to read headers
       def self.from_native(native_message)
-        headers_ptrptr = FFI::MemoryPointer.new(:pointer)
+        # Scratch output pointers for the header FFI calls, reused across messages. They are
+        # stored in fiber-local storage (`Thread.current[]` is fiber-local by design), so each
+        # thread and each fiber gets its own set and reuse is safe with fiber schedulers. They
+        # never escape this method and all data is read out of them before returning.
+        # Allocating them per message would cost several native allocations for every consumed
+        # message.
+        headers_ptrptr, name_ptrptr, value_ptrptr, size_ptr =
+          Thread.current[SCRATCH_KEY] ||= [
+            FFI::MemoryPointer.new(:pointer),
+            FFI::MemoryPointer.new(:pointer),
+            FFI::MemoryPointer.new(:pointer),
+            Rdkafka::Bindings::SizePtr.new
+          ].freeze
+
         err = Rdkafka::Bindings.rd_kafka_message_headers(native_message, headers_ptrptr)
 
         if err == Rdkafka::Bindings::RD_KAFKA_RESP_ERR__NOENT
@@ -28,10 +46,6 @@ module Rdkafka
 
         headers_ptr = headers_ptrptr.read_pointer
 
-        name_ptrptr = FFI::MemoryPointer.new(:pointer)
-        value_ptrptr = FFI::MemoryPointer.new(:pointer)
-        size_ptr = Rdkafka::Bindings::SizePtr.new
-
         headers = {}
 
         idx = 0

data/lib/rdkafka/consumer/partition.rb

@@ -16,14 +16,20 @@ module Rdkafka
       # @return [Integer]
       attr_reader :err
 
+      # Partition metadata in the context of a consumer
+      # @return [String, nil]
+      attr_reader :metadata
+
       # @private
       # @param partition [Integer] partition number
       # @param offset [Integer, nil] partition offset
       # @param err [Integer] error code from librdkafka
-      def initialize(partition, offset, err = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR)
+      # @param metadata [String, nil] partition metadata
+      def initialize(partition, offset, err = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR, metadata = nil)
         @partition = partition
         @offset = offset
         @err = err
+        @metadata = metadata
       end
 
       # Human readable representation of this partition.
@@ -32,6 +38,7 @@ module Rdkafka
         message = "<Partition #{partition}"
         message += " offset=#{offset}" if offset
         message += " err=#{err}" if err != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+        message += " metadata=#{metadata}" if !metadata.nil?
         message += ">"
         message
       end