rdkafka 0.28.0-aarch64-linux-gnu → 0.29.1-aarch64-linux-gnu

data/lib/rdkafka/callbacks/list_offsets_handler.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Callbacks
+    # Handles `RD_KAFKA_EVENT_LISTOFFSETS_RESULT` events
+    # @private
+    class ListOffsetsHandler < BaseHandler
+      class << self
+        # Resolves the list-offsets handle from its result event
+        # @param event_ptr [FFI::Pointer] pointer to the event
+        # @return [void]
+        def call(event_ptr)
+          list_offsets = ListOffsetsResult.new(event_ptr)
+          list_offsets_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+          if list_offsets_handle = Rdkafka::Admin::ListOffsetsHandle.remove(list_offsets_handle_ptr.address)
+            list_offsets_handle[:response] = list_offsets.result_error
+            list_offsets_handle.broker_message = read_event_string(list_offsets.error_string)
+
+            # Parsing can raise on per-partition errors. The exception is captured and re-raised
+            # on the waiting thread, since an exception leaving this callback would unwind through
+            # librdkafka native frames.
+            list_offsets_handle.result = begin
+              if list_offsets.result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+                Rdkafka::Admin::ListOffsetsReport.new(
+                  result_infos: list_offsets.result_infos,
+                  result_count: list_offsets.result_count
+                )
+              else
+                Rdkafka::Admin::ListOffsetsReport.new(result_infos: FFI::Pointer::NULL, result_count: 0)
+              end
+            rescue => e
+              e
+            end
+
+            list_offsets_handle.unlock
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/callbacks.rb

@@ -98,7 +98,7 @@ module Rdkafka
         @error_string = Rdkafka::Bindings.rd_kafka_error_string(rd_kafka_error_pointer)
         if @result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
           # Get the number of matching acls
-          pointer_to_size_t = FFI::MemoryPointer.new(:int32)
+          pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
           @matching_acls = Rdkafka::Bindings.rd_kafka_DeleteAcls_result_response_matching_acls(acl_result_pointer, pointer_to_size_t)
           @matching_acls_count = pointer_to_size_t.read_int
         end
@@ -129,7 +129,7 @@ module Rdkafka
         if @result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
           acl_describe_result = Rdkafka::Bindings.rd_kafka_event_DescribeAcls_result(event_ptr)
           # Get the number of matching acls
-          pointer_to_size_t = FFI::MemoryPointer.new(:int32)
+          pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
           @matching_acls = Rdkafka::Bindings.rd_kafka_DescribeAcls_result_acls(acl_describe_result, pointer_to_size_t)
           @matching_acls_count = pointer_to_size_t.read_int
         end
@@ -151,7 +151,7 @@ module Rdkafka
         if @result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
           configs_describe_result = Rdkafka::Bindings.rd_kafka_event_DescribeConfigs_result(event_ptr)
           # Get the number of matching acls
-          pointer_to_size_t = FFI::MemoryPointer.new(:int32)
+          pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
           @results = Rdkafka::Bindings.rd_kafka_DescribeConfigs_result_resources(configs_describe_result, pointer_to_size_t)
           @results_count = pointer_to_size_t.read_int
         end
@@ -173,7 +173,7 @@ module Rdkafka
         if @result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
           incremental_alter_result = Rdkafka::Bindings.rd_kafka_event_IncrementalAlterConfigs_result(event_ptr)
           # Get the number of matching acls
-          pointer_to_size_t = FFI::MemoryPointer.new(:int32)
+          pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
           @results = Rdkafka::Bindings.rd_kafka_IncrementalAlterConfigs_result_resources(incremental_alter_result, pointer_to_size_t)
           @results_count = pointer_to_size_t.read_int
         end
@@ -208,232 +208,28 @@ module Rdkafka
       # @param event_ptr [FFI::Pointer] pointer to the event
       # @param _opaque_ptr [FFI::Pointer] unused opaque pointer
       def self.call(_client_ptr, event_ptr, _opaque_ptr)
-        case Rdkafka::Bindings.rd_kafka_event_type(event_ptr)
-        when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATETOPICS_RESULT
-          process_create_topic(event_ptr)
-        when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBECONFIGS_RESULT
-          process_describe_configs(event_ptr)
-        when Rdkafka::Bindings::RD_KAFKA_EVENT_INCREMENTALALTERCONFIGS_RESULT
-          process_incremental_alter_configs(event_ptr)
-        when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETETOPICS_RESULT
-          process_delete_topic(event_ptr)
-        when Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS_RESULT
-          process_create_partitions(event_ptr)
-        when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATEACLS_RESULT
-          process_create_acl(event_ptr)
-        when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEACLS_RESULT
-          process_delete_acl(event_ptr)
-        when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBEACLS_RESULT
-          process_describe_acl(event_ptr)
-        when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEGROUPS_RESULT
-          process_delete_groups(event_ptr)
-        when Rdkafka::Bindings::RD_KAFKA_EVENT_LISTOFFSETS_RESULT
-          process_list_offsets(event_ptr)
-        end
-      end
-
-      private
-
-      # Processes create topic result event
-      # @param event_ptr [FFI::Pointer] pointer to the event
-      def self.process_create_topic(event_ptr)
-        create_topics_result = Rdkafka::Bindings.rd_kafka_event_CreateTopics_result(event_ptr)
-
-        # Get the number of create topic results
-        pointer_to_size_t = FFI::MemoryPointer.new(:int32)
-        create_topic_result_array = Rdkafka::Bindings.rd_kafka_CreateTopics_result_topics(create_topics_result, pointer_to_size_t)
-        create_topic_results = TopicResult.create_topic_results_from_array(pointer_to_size_t.read_int, create_topic_result_array)
-        create_topic_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
-
-        if create_topic_handle = Rdkafka::Admin::CreateTopicHandle.remove(create_topic_handle_ptr.address)
-          create_topic_handle[:response] = create_topic_results[0].result_error
-          create_topic_handle[:error_string] = create_topic_results[0].error_string
-          create_topic_handle[:result_name] = create_topic_results[0].result_name
-
-          create_topic_handle.unlock
-        end
-      end
-
-      # Processes describe configs result event
-      # @param event_ptr [FFI::Pointer] pointer to the event
-      def self.process_describe_configs(event_ptr)
-        describe_configs = DescribeConfigsResult.new(event_ptr)
-        describe_configs_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
-
-        if describe_configs_handle = Rdkafka::Admin::DescribeConfigsHandle.remove(describe_configs_handle_ptr.address)
-          describe_configs_handle[:response] = describe_configs.result_error
-          describe_configs_handle[:response_string] = describe_configs.error_string
-          describe_configs_handle[:pending] = false
-
-          if describe_configs.result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
-            describe_configs_handle[:config_entries] = describe_configs.results
-            describe_configs_handle[:entry_count] = describe_configs.results_count
-          end
-
-          describe_configs_handle.unlock
-        end
-      end
-
-      # Processes incremental alter configs result event
-      # @param event_ptr [FFI::Pointer] pointer to the event
-      def self.process_incremental_alter_configs(event_ptr)
-        incremental_alter = IncrementalAlterConfigsResult.new(event_ptr)
-        incremental_alter_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
-
-        if incremental_alter_handle = Rdkafka::Admin::IncrementalAlterConfigsHandle.remove(incremental_alter_handle_ptr.address)
-          incremental_alter_handle[:response] = incremental_alter.result_error
-          incremental_alter_handle[:response_string] = incremental_alter.error_string
-          incremental_alter_handle[:pending] = false
-
-          if incremental_alter.result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
-            incremental_alter_handle[:config_entries] = incremental_alter.results
-            incremental_alter_handle[:entry_count] = incremental_alter.results_count
-          end
-
-          incremental_alter_handle.unlock
-        end
-      end
-
-      # Processes delete groups result event
-      # @param event_ptr [FFI::Pointer] pointer to the event
-      def self.process_delete_groups(event_ptr)
-        delete_groups_result = Rdkafka::Bindings.rd_kafka_event_DeleteGroups_result(event_ptr)
-
-        # Get the number of delete group results
-        pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
-        delete_group_result_array = Rdkafka::Bindings.rd_kafka_DeleteGroups_result_groups(delete_groups_result, pointer_to_size_t)
-        delete_group_results = GroupResult.create_group_results_from_array(pointer_to_size_t.read_int, delete_group_result_array) # TODO fix this
-        delete_group_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
-
-        if (delete_group_handle = Rdkafka::Admin::DeleteGroupsHandle.remove(delete_group_handle_ptr.address))
-          delete_group_handle[:response] = delete_group_results[0].result_error
-          delete_group_handle[:error_string] = delete_group_results[0].error_string
-          delete_group_handle[:result_name] = delete_group_results[0].result_name
-
-          delete_group_handle.unlock
-        end
-      end
-
-      # Processes delete topic result event
-      # @param event_ptr [FFI::Pointer] pointer to the event
-      def self.process_delete_topic(event_ptr)
-        delete_topics_result = Rdkafka::Bindings.rd_kafka_event_DeleteTopics_result(event_ptr)
-
-        # Get the number of topic results
-        pointer_to_size_t = FFI::MemoryPointer.new(:int32)
-        delete_topic_result_array = Rdkafka::Bindings.rd_kafka_DeleteTopics_result_topics(delete_topics_result, pointer_to_size_t)
-        delete_topic_results = TopicResult.create_topic_results_from_array(pointer_to_size_t.read_int, delete_topic_result_array)
-        delete_topic_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
-
-        if delete_topic_handle = Rdkafka::Admin::DeleteTopicHandle.remove(delete_topic_handle_ptr.address)
-          delete_topic_handle[:response] = delete_topic_results[0].result_error
-          delete_topic_handle[:error_string] = delete_topic_results[0].error_string
-          delete_topic_handle[:result_name] = delete_topic_results[0].result_name
-
-          delete_topic_handle.unlock
+        handler = case Rdkafka::Bindings.rd_kafka_event_type(event_ptr)
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATETOPICS_RESULT then CreateTopicHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETETOPICS_RESULT then DeleteTopicHandler
+        when Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS_RESULT then CreatePartitionsHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEGROUPS_RESULT then DeleteGroupsHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETERECORDS_RESULT then DeleteRecordsHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATEACLS_RESULT then CreateAclHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEACLS_RESULT then DeleteAclHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBEACLS_RESULT then DescribeAclHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBECONFIGS_RESULT then DescribeConfigsHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_INCREMENTALALTERCONFIGS_RESULT then IncrementalAlterConfigsHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_LISTOFFSETS_RESULT then ListOffsetsHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_LISTCONSUMERGROUPS_RESULT then ListConsumerGroupsHandler
         end
-      end
-
-      # Processes create partitions result event
-      # @param event_ptr [FFI::Pointer] pointer to the event
-      def self.process_create_partitions(event_ptr)
-        create_partitionss_result = Rdkafka::Bindings.rd_kafka_event_CreatePartitions_result(event_ptr)
-
-        # Get the number of create topic results
-        pointer_to_size_t = FFI::MemoryPointer.new(:int32)
-        create_partitions_result_array = Rdkafka::Bindings.rd_kafka_CreatePartitions_result_topics(create_partitionss_result, pointer_to_size_t)
-        create_partitions_results = TopicResult.create_topic_results_from_array(pointer_to_size_t.read_int, create_partitions_result_array)
-        create_partitions_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
 
-        if create_partitions_handle = Rdkafka::Admin::CreatePartitionsHandle.remove(create_partitions_handle_ptr.address)
-          create_partitions_handle[:response] = create_partitions_results[0].result_error
-          create_partitions_handle[:error_string] = create_partitions_results[0].error_string
-          create_partitions_handle[:result_name] = create_partitions_results[0].result_name
-
-          create_partitions_handle.unlock
-        end
-      end
-
-      # Processes create ACL result event
-      # @param event_ptr [FFI::Pointer] pointer to the event
-      def self.process_create_acl(event_ptr)
-        create_acls_result = Rdkafka::Bindings.rd_kafka_event_CreateAcls_result(event_ptr)
-
-        # Get the number of acl results
-        pointer_to_size_t = FFI::MemoryPointer.new(:int32)
-        create_acl_result_array = Rdkafka::Bindings.rd_kafka_CreateAcls_result_acls(create_acls_result, pointer_to_size_t)
-        create_acl_results = CreateAclResult.create_acl_results_from_array(pointer_to_size_t.read_int, create_acl_result_array)
-        create_acl_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
-
-        if create_acl_handle = Rdkafka::Admin::CreateAclHandle.remove(create_acl_handle_ptr.address)
-          create_acl_handle[:response] = create_acl_results[0].result_error
-          create_acl_handle[:response_string] = create_acl_results[0].error_string
-
-          create_acl_handle.unlock
-        end
-      end
-
-      # Processes delete ACL result event
-      # @param event_ptr [FFI::Pointer] pointer to the event
-      def self.process_delete_acl(event_ptr)
-        delete_acls_result = Rdkafka::Bindings.rd_kafka_event_DeleteAcls_result(event_ptr)
-
-        # Get the number of acl results
-        pointer_to_size_t = FFI::MemoryPointer.new(:int32)
-        delete_acl_result_responses = Rdkafka::Bindings.rd_kafka_DeleteAcls_result_responses(delete_acls_result, pointer_to_size_t)
-        delete_acl_results = DeleteAclResult.delete_acl_results_from_array(pointer_to_size_t.read_int, delete_acl_result_responses)
-        delete_acl_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
-
-        if delete_acl_handle = Rdkafka::Admin::DeleteAclHandle.remove(delete_acl_handle_ptr.address)
-          delete_acl_handle[:response] = delete_acl_results[0].result_error
-          delete_acl_handle[:response_string] = delete_acl_results[0].error_string
-
-          if delete_acl_results[0].result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
-            delete_acl_handle[:matching_acls] = delete_acl_results[0].matching_acls
-            delete_acl_handle[:matching_acls_count] = delete_acl_results[0].matching_acls_count
-          end
-
-          delete_acl_handle.unlock
-        end
-      end
-
-      # Processes describe ACL result event
-      # @param event_ptr [FFI::Pointer] pointer to the event
-      def self.process_describe_acl(event_ptr)
-        describe_acl = DescribeAclResult.new(event_ptr)
-        describe_acl_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
-
-        if describe_acl_handle = Rdkafka::Admin::DescribeAclHandle.remove(describe_acl_handle_ptr.address)
-          describe_acl_handle[:response] = describe_acl.result_error
-          describe_acl_handle[:response_string] = describe_acl.error_string
-
-          if describe_acl.result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
-            describe_acl_handle[:acls] = describe_acl.matching_acls
-            describe_acl_handle[:acls_count] = describe_acl.matching_acls_count
-          end
-
-          describe_acl_handle.unlock
-        end
-      end
-
-      # Processes list offsets result event
-      # @param event_ptr [FFI::Pointer] pointer to the event
-      def self.process_list_offsets(event_ptr)
-        list_offsets = ListOffsetsResult.new(event_ptr)
-        list_offsets_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
-
-        if list_offsets_handle = Rdkafka::Admin::ListOffsetsHandle.remove(list_offsets_handle_ptr.address)
-          list_offsets_handle[:response] = list_offsets.result_error
-          list_offsets_handle[:response_string] = list_offsets.error_string
-          list_offsets_handle[:pending] = false
-
-          if list_offsets.result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
-            list_offsets_handle[:result_infos] = list_offsets.result_infos
-            list_offsets_handle[:result_count] = list_offsets.result_count
-          end
-
-          list_offsets_handle.unlock
-        end
+        handler&.call(event_ptr)
+      ensure
+        # Background queue events are owned by the application and must be destroyed once
+        # served (see rd_kafka_conf_set_background_event_cb). All event-owned memory (result
+        # arrays, strings) has to be copied into Ruby objects by the handlers above before this
+        # runs - nothing may retain pointers into the event past this point.
+        Rdkafka::Bindings.rd_kafka_event_destroy(event_ptr)
       end
     end
 
@@ -453,30 +249,42 @@ module Rdkafka
           delivery_handle[:response] = message[:err]
           delivery_handle[:partition] = message[:partition]
           delivery_handle[:offset] = message[:offset]
-          delivery_handle[:topic_name] = FFI::MemoryPointer.from_string(topic_name)
-
-          # Call delivery callback on opaque
-          if opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
-            opaque.call_delivery_callback(
-              Rdkafka::Producer::DeliveryReport.new(
-                message[:partition],
-                message[:offset],
-                topic_name,
-                message[:err],
-                delivery_handle.label
-              ),
-              delivery_handle
-            )
+          # The topic is not stored on the handle here: it is already set as the handle's
+          # `topic` Ruby attribute during produce, which spares a native string copy that
+          # would otherwise be allocated and retained for every message.
+
+          begin
+            # Call delivery callback on opaque
+            if opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
+              opaque.call_delivery_callback(
+                Rdkafka::Producer::DeliveryReport.new(
+                  message[:partition],
+                  message[:offset],
+                  topic_name,
+                  message[:err],
+                  delivery_handle.label
+                ),
+                delivery_handle
+              )
+            end
+          rescue Exception => err
+            # A user delivery callback that raises must not escape this FFI callback: it runs on
+            # librdkafka's polling thread (abort_on_exception = true), so an escaping exception
+            # would crash the whole process. We log and swallow it, matching the rebalance
+            # callback. The `ensure` below still unlocks the handle, so `wait` returns the
+            # delivery report instead of blocking until its timeout and raising
+            # WaitTimeoutError for a message that was in fact delivered.
+            Rdkafka::Config.logger.error("Unhandled exception: #{err.class} - #{err.message}")
+          ensure
+            delivery_handle.unlock
           end
-
-          delivery_handle.unlock
         end
       end
     end
 
-    # @private
+    # @!visibility private
     @@mutex = Mutex.new
-    # @private
+    # @!visibility private
     @@current_pid = nil
 
     class << self

data/lib/rdkafka/config.rb

@@ -3,24 +3,24 @@
 module Rdkafka
   # Configuration for a Kafka consumer or producer. You can create an instance and use
   # the consumer and producer methods to create a client. Documentation of the available
-  # configuration options is available on https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md.
+  # configuration options is available on
+  # https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md.
   class Config
-    # @private
+    # @!visibility private
     @@logger = Logger.new($stdout)
-    # @private
+    # @!visibility private
     @@statistics_callback = nil
-    # @private
+    # @!visibility private
     @@error_callback = nil
-    # @private
+    # @!visibility private
     @@opaques = ObjectSpace::WeakMap.new
-    # @private
+    # @!visibility private
     @@log_queue = Queue.new
-    # We memoize thread on the first log flush
-    # This allows us also to restart logger thread on forks
+    # We memoize thread on first log flush. This allows us also to restart logger thread on forks
     @@log_thread = nil
-    # @private
+    # @!visibility private
     @@log_mutex = Mutex.new
-    # @private
+    # @!visibility private
     @@oauthbearer_token_refresh_callback = nil
 
     # Returns the current logger, by default this is a logger to stdout.
@@ -69,7 +69,8 @@ module Rdkafka
 
     # Set a callback that will be called every time the underlying client emits statistics.
     # You can configure if and how often this happens using `statistics.interval.ms`.
-    # The callback is called with a hash that's documented here: https://github.com/confluentinc/librdkafka/blob/master/STATISTICS.md
+    # The callback is called with a hash that's documented here:
+    # https://github.com/confluentinc/librdkafka/blob/master/STATISTICS.md
     #
     # @param callback [Proc, #call, nil] callable object or nil to clear
     # @return [nil]
@@ -86,7 +87,8 @@ module Rdkafka
     end
 
     # Set a callback that will be called every time the underlying client emits an error.
-    # If this callback is not set, global errors such as brokers becoming unavailable will only be sent to the logger, as defined by librdkafka.
+    # If this callback is not set, global errors such as brokers becoming unavailable will only be
+    # sent to the logger, as defined by librdkafka.
     # The callback is called with an instance of RdKafka::Error.
     #
     # @param callback [Proc, #call, nil] callable object to handle errors or nil to clear
@@ -197,6 +199,11 @@ module Rdkafka
       opaque = Opaque.new
       config = native_config(opaque)
 
+      # Set callback to receive background events, so admin-style operations issued on the
+      # consumer handle (e.g. Consumer#list_offsets) get their results dispatched back to their
+      # handles. librdkafka spawns its internally-managed background thread because of this.
+      Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
+
       if @consumer_rebalance_listener
         opaque.consumer_rebalance_listener = @consumer_rebalance_listener
         Rdkafka::Bindings.rd_kafka_conf_set_rebalance_cb(config, Rdkafka::Bindings::RebalanceCallback)
@@ -290,6 +297,9 @@ module Rdkafka
     # Uses `rd_kafka_conf_dump` to retrieve every property (including defaults and
     # internal properties like `client.software.name`) as a flat Hash.
     #
+    # @return [Hash{Symbol => String}] property names mapped to their current values
+    #
+    # @raise [ConfigError] When the configuration contains invalid options
     # @note The librdkafka C API does not distinguish between producer-only, consumer-only,
     #   and global properties at the configuration level. All properties are returned
     #   regardless of the intended client type.
@@ -297,10 +307,6 @@ module Rdkafka
     # @note The returned Hash may include sensitive values such as authentication
     #   credentials and key passwords. Do not log or serialize the returned data
     #   unless you have explicitly redacted secret entries.
-    #
-    # @return [Hash{Symbol => String}] property names mapped to their current values
-    #
-    # @raise [ConfigError] When the configuration contains invalid options
     def describe_properties
       config = nil
       dump_ptr = nil
@@ -325,7 +331,8 @@ module Rdkafka
       Rdkafka::Bindings.rd_kafka_conf_destroy(config) if config
     end
 
-    # Error that is returned by the underlying rdkafka error if an invalid configuration option is present.
+    # Error that is returned by the underlying rdkafka error if an invalid configuration option is
+    # present.
     class ConfigError < RuntimeError; end
 
     # Error that is returned by the underlying rdkafka library if the client cannot be created.
@@ -342,47 +349,55 @@ module Rdkafka
     # @param opaque [Object, nil] optional opaque pointer for callbacks
     # @return [FFI::Pointer] native rdkafka configuration pointer
     def native_config(opaque = nil)
-      Rdkafka::Bindings.rd_kafka_conf_new.tap do |config|
-        # Create config
-        @config_hash.merge(REQUIRED_CONFIG).each do |key, value|
-          error_buffer = FFI::MemoryPointer.from_string(" " * 256)
-          result = Rdkafka::Bindings.rd_kafka_conf_set(
-            config,
-            key.to_s,
-            value.to_s,
-            error_buffer,
-            256
-          )
-          unless result == :config_ok
-            raise ConfigError.new(error_buffer.read_string)
-          end
+      config = Rdkafka::Bindings.rd_kafka_conf_new
+
+      # Create config
+      @config_hash.merge(REQUIRED_CONFIG).each do |key, value|
+        error_buffer = FFI::MemoryPointer.from_string(" " * 256)
+        result = Rdkafka::Bindings.rd_kafka_conf_set(
+          config,
+          key.to_s,
+          value.to_s,
+          error_buffer,
+          256
+        )
+        unless result == :config_ok
+          raise ConfigError.new(error_buffer.read_string)
         end
+      end
 
-        # Set opaque pointer that's used as a proxy for callbacks
-        if opaque
-          pointer = ::FFI::Pointer.new(:pointer, opaque.object_id)
-          Rdkafka::Bindings.rd_kafka_conf_set_opaque(config, pointer)
+      # Set opaque pointer that's used as a proxy for callbacks
+      if opaque
+        pointer = ::FFI::Pointer.new(:pointer, opaque.object_id)
+        Rdkafka::Bindings.rd_kafka_conf_set_opaque(config, pointer)
 
-          # Store opaque with the pointer as key. We use this approach instead
-          # of trying to convert the pointer to a Ruby object because there is
-          # no risk of a segfault this way.
-          Rdkafka::Config.opaques[pointer.to_i] = opaque
-        end
+        # Store opaque with the pointer as key. We use this approach instead
+        # of trying to convert the pointer to a Ruby object because there is
+        # no risk of a segfault this way.
+        Rdkafka::Config.opaques[pointer.to_i] = opaque
+      end
 
-        # Set log callback
-        Rdkafka::Bindings.rd_kafka_conf_set_log_cb(config, Rdkafka::Bindings::LogCallback)
+      # Set log callback
+      Rdkafka::Bindings.rd_kafka_conf_set_log_cb(config, Rdkafka::Bindings::LogCallback)
 
-        # Set stats callback
-        Rdkafka::Bindings.rd_kafka_conf_set_stats_cb(config, Rdkafka::Bindings::StatsCallback)
+      # Set stats callback
+      Rdkafka::Bindings.rd_kafka_conf_set_stats_cb(config, Rdkafka::Bindings::StatsCallback)
 
-        # Set error callback
-        Rdkafka::Bindings.rd_kafka_conf_set_error_cb(config, Rdkafka::Bindings::ErrorCallback)
+      # Set error callback
+      Rdkafka::Bindings.rd_kafka_conf_set_error_cb(config, Rdkafka::Bindings::ErrorCallback)
 
-        # Set oauth callback
-        if Rdkafka::Config.oauthbearer_token_refresh_callback
-          Rdkafka::Bindings.rd_kafka_conf_set_oauthbearer_token_refresh_cb(config, Rdkafka::Bindings::OAuthbearerTokenRefreshCallback)
-        end
+      # Set oauth callback
+      if Rdkafka::Config.oauthbearer_token_refresh_callback
+        Rdkafka::Bindings.rd_kafka_conf_set_oauthbearer_token_refresh_cb(config, Rdkafka::Bindings::OAuthbearerTokenRefreshCallback)
       end
+
+      config
+    rescue Exception
+      # rd_kafka_conf_new allocates a conf the caller owns until it is handed to rd_kafka_new. If
+      # we raise before returning it (e.g. an invalid option in the loop above), destroy it here
+      # so the native config is not leaked.
+      Rdkafka::Bindings.rd_kafka_conf_destroy(config) if config
+      raise
     end
 
     # Creates a native Kafka handle
@@ -400,6 +415,10 @@ module Rdkafka
       )
 
       if handle.null?
+        # On success rd_kafka_new takes ownership of the conf and frees it itself; on failure
+        # librdkafka keeps application ownership (see rdkafka.c), so we must destroy it here to
+        # avoid leaking the conf on every failed client creation.
+        Rdkafka::Bindings.rd_kafka_conf_destroy(config)
         raise ClientCreationError.new(error_buffer.read_string)
       end
 

data/lib/rdkafka/consumer/headers.rb

@@ -7,6 +7,11 @@ module Rdkafka
       # Empty frozen hash used when there are no headers
       EMPTY_HEADERS = {}.freeze
 
+      # Key under which reusable scratch pointers are stored on the current thread/fiber
+      SCRATCH_KEY = :rdkafka_headers_scratch
+
+      private_constant :SCRATCH_KEY
+
       # Reads a librdkafka native message's headers and returns them as a Ruby Hash
       # where each key maps to either a String (single value) or Array<String> (multiple values)
       # to support duplicate headers per KIP-82
@@ -17,7 +22,20 @@ module Rdkafka
       # @return [Hash{String => String, Array<String>}] headers Hash for the native_message
       # @raise [Rdkafka::RdkafkaError] when fail to read headers
       def self.from_native(native_message)
-        headers_ptrptr = FFI::MemoryPointer.new(:pointer)
+        # Scratch output pointers for the header FFI calls, reused across messages. They are
+        # stored in fiber-local storage (`Thread.current[]` is fiber-local by design), so each
+        # thread and each fiber gets its own set and reuse is safe with fiber schedulers. They
+        # never escape this method and all data is read out of them before returning.
+        # Allocating them per message would cost several native allocations for every consumed
+        # message.
+        headers_ptrptr, name_ptrptr, value_ptrptr, size_ptr =
+          Thread.current[SCRATCH_KEY] ||= [
+            FFI::MemoryPointer.new(:pointer),
+            FFI::MemoryPointer.new(:pointer),
+            FFI::MemoryPointer.new(:pointer),
+            Rdkafka::Bindings::SizePtr.new
+          ].freeze
+
         err = Rdkafka::Bindings.rd_kafka_message_headers(native_message, headers_ptrptr)
 
         if err == Rdkafka::Bindings::RD_KAFKA_RESP_ERR__NOENT
@@ -28,10 +46,6 @@ module Rdkafka
 
         headers_ptr = headers_ptrptr.read_pointer
 
-        name_ptrptr = FFI::MemoryPointer.new(:pointer)
-        value_ptrptr = FFI::MemoryPointer.new(:pointer)
-        size_ptr = Rdkafka::Bindings::SizePtr.new
-
         headers = {}
 
         idx = 0

data/lib/rdkafka/consumer/partition.rb

@@ -16,14 +16,20 @@ module Rdkafka
       # @return [Integer]
       attr_reader :err
 
+      # Partition metadata in the context of a consumer
+      # @return [String, nil]
+      attr_reader :metadata
+
       # @private
       # @param partition [Integer] partition number
       # @param offset [Integer, nil] partition offset
       # @param err [Integer] error code from librdkafka
-      def initialize(partition, offset, err = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR)
+      # @param metadata [String, nil] partition metadata
+      def initialize(partition, offset, err = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR, metadata = nil)
         @partition = partition
         @offset = offset
         @err = err
+        @metadata = metadata
       end
 
       # Human readable representation of this partition.
@@ -32,6 +38,7 @@ module Rdkafka
         message = "<Partition #{partition}"
         message += " offset=#{offset}" if offset
         message += " err=#{err}" if err != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+        message += " metadata=#{metadata}" if !metadata.nil?
         message += ">"
         message
       end