rdkafka 0.22.2 → 0.29.0

data/lib/rdkafka/config.rb

@@ -6,7 +6,7 @@ module Rdkafka
   # configuration options is available on https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md.
   class Config
     # @private
-    @@logger = Logger.new(STDOUT)
+    @@logger = Logger.new($stdout)
     # @private
     @@statistics_callback = nil
     # @private
@@ -33,7 +33,7 @@ module Rdkafka
     # Makes sure that there is a thread for consuming logs
     # We do not spawn thread immediately and we need to check if it operates to support forking
     def self.ensure_log_thread
-      return if @@log_thread && @@log_thread.alive?
+      return if @@log_thread&.alive?
 
       @@log_mutex.synchronize do
         # Restart if dead (fork, crash)
@@ -71,11 +71,10 @@ module Rdkafka
     # You can configure if and how often this happens using `statistics.interval.ms`.
     # The callback is called with a hash that's documented here: https://github.com/confluentinc/librdkafka/blob/master/STATISTICS.md
     #
-    # @param callback [Proc, #call] The callback
-    #
+    # @param callback [Proc, #call, nil] callable object or nil to clear
     # @return [nil]
     def self.statistics_callback=(callback)
-      raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call) || callback == nil
+      raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call) || callback.nil?
       @@statistics_callback = callback
     end
 
@@ -90,11 +89,10 @@ module Rdkafka
     # If this callback is not set, global errors such as brokers becoming unavailable will only be sent to the logger, as defined by librdkafka.
     # The callback is called with an instance of RdKafka::Error.
     #
-    # @param callback [Proc, #call] The callback
-    #
+    # @param callback [Proc, #call, nil] callable object to handle errors or nil to clear
     # @return [nil]
     def self.error_callback=(callback)
-      raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call)
+      raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call) || callback.nil?
       @@error_callback = callback
     end
 
@@ -108,11 +106,10 @@ module Rdkafka
     # Sets the SASL/OAUTHBEARER token refresh callback.
     # This callback will be triggered when it is time to refresh the client's OAUTHBEARER token
     #
-    # @param callback [Proc, #call] The callback
-    #
+    # @param callback [Proc, #call, nil] callable object to handle token refresh or nil to clear
     # @return [nil]
     def self.oauthbearer_token_refresh_callback=(callback)
-      raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call) || callback == nil
+      raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call) || callback.nil?
       @@oauthbearer_token_refresh_callback = callback
     end
 
@@ -129,15 +126,12 @@ module Rdkafka
     end
 
     # Default config that can be overwritten.
-    DEFAULT_CONFIG = {
-      # Request api version so advanced features work
-      :"api.version.request" => true
-    }.freeze
+    DEFAULT_CONFIG = {}.freeze
 
     # Required config that cannot be overwritten.
     REQUIRED_CONFIG = {
       # Enable log queues so we get callbacks in our own Ruby threads
-      :"log.queue" => true
+      "log.queue": true
     }.freeze
 
     # Returns a new config with the provided options which are merged with {DEFAULT_CONFIG}.
@@ -174,10 +168,8 @@ module Rdkafka
 
     # Get notifications on partition assignment/revocation for the subscribed topics
     #
-    # @param listener [Object, #on_partitions_assigned, #on_partitions_revoked] listener instance
-    def consumer_rebalance_listener=(listener)
-      @consumer_rebalance_listener = listener
-    end
+    # @return [Object, #on_partitions_assigned, #on_partitions_revoked] listener instance
+    attr_writer :consumer_rebalance_listener
 
     # Should we use a single queue for the underlying consumer and events.
     #
@@ -190,10 +182,8 @@ module Rdkafka
     # It is recommended to use the defaults and only set it to `false` in advance multi-threaded
     # and complex cases where granular events handling control is needed.
     #
-    # @param poll_set [Boolean]
-    def consumer_poll_set=(poll_set)
-      @consumer_poll_set = poll_set
-    end
+    # @return [Boolean]
+    attr_writer :consumer_poll_set
 
     # Creates a consumer with this configuration.
     #
@@ -234,11 +224,13 @@ module Rdkafka
     # @param native_kafka_auto_start [Boolean] should the native kafka operations be started
     #   automatically. Defaults to true. Set to false only when doing complex initialization.
     # @param native_kafka_poll_timeout_ms [Integer] ms poll time of the native Kafka
+    # @param run_polling_thread [Boolean] should the background polling thread be started.
+    #   Defaults to true. Set to false when using the FD API for fiber scheduler integration.
     # @return [Producer] The created producer
     #
     # @raise [ConfigError] When the configuration contains invalid options
     # @raise [ClientCreationError] When the native client cannot be created
-    def producer(native_kafka_auto_start: true, native_kafka_poll_timeout_ms: 100)
+    def producer(native_kafka_auto_start: true, native_kafka_poll_timeout_ms: Defaults::NATIVE_KAFKA_POLL_TIMEOUT_MS, run_polling_thread: true)
       # Create opaque
       opaque = Opaque.new
       # Create Kafka config
@@ -253,7 +245,7 @@ module Rdkafka
       Rdkafka::Producer.new(
         Rdkafka::NativeKafka.new(
           kafka,
-          run_polling_thread: true,
+          run_polling_thread: run_polling_thread,
           opaque: opaque,
           auto_start: native_kafka_auto_start,
           timeout_ms: native_kafka_poll_timeout_ms
@@ -269,11 +261,13 @@ module Rdkafka
     # @param native_kafka_auto_start [Boolean] should the native kafka operations be started
     #   automatically. Defaults to true. Set to false only when doing complex initialization.
     # @param native_kafka_poll_timeout_ms [Integer] ms poll time of the native Kafka
+    # @param run_polling_thread [Boolean] should the background polling thread be started.
+    #   Defaults to true. Set to false when using the FD API for fiber scheduler integration.
     # @return [Admin] The created admin instance
     #
     # @raise [ConfigError] When the configuration contains invalid options
     # @raise [ClientCreationError] When the native client cannot be created
-    def admin(native_kafka_auto_start: true, native_kafka_poll_timeout_ms: 100)
+    def admin(native_kafka_auto_start: true, native_kafka_poll_timeout_ms: Defaults::NATIVE_KAFKA_POLL_TIMEOUT_MS, run_polling_thread: true)
       opaque = Opaque.new
       config = native_config(opaque)
       Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
@@ -283,7 +277,7 @@ module Rdkafka
       Rdkafka::Admin.new(
         Rdkafka::NativeKafka.new(
           kafka,
-          run_polling_thread: true,
+          run_polling_thread: run_polling_thread,
           opaque: opaque,
           auto_start: native_kafka_auto_start,
           timeout_ms: native_kafka_poll_timeout_ms
@@ -291,6 +285,46 @@ module Rdkafka
       )
     end
 
+    # Returns all configuration properties and their current values for this config.
+    #
+    # Uses `rd_kafka_conf_dump` to retrieve every property (including defaults and
+    # internal properties like `client.software.name`) as a flat Hash.
+    #
+    # @note The librdkafka C API does not distinguish between producer-only, consumer-only,
+    #   and global properties at the configuration level. All properties are returned
+    #   regardless of the intended client type.
+    #
+    # @note The returned Hash may include sensitive values such as authentication
+    #   credentials and key passwords. Do not log or serialize the returned data
+    #   unless you have explicitly redacted secret entries.
+    #
+    # @return [Hash{Symbol => String}] property names mapped to their current values
+    #
+    # @raise [ConfigError] When the configuration contains invalid options
+    def describe_properties
+      config = nil
+      dump_ptr = nil
+      count = 0
+
+      config = native_config
+      count_ptr = Rdkafka::Bindings::SizePtr.new
+      dump_ptr = Rdkafka::Bindings.rd_kafka_conf_dump(config, count_ptr)
+
+      count = count_ptr[:value]
+      result = {}
+
+      (0...count).step(2) do |i|
+        key = dump_ptr.get_pointer(i * FFI::Pointer.size).read_string
+        value = dump_ptr.get_pointer((i + 1) * FFI::Pointer.size).read_string
+        result[key.to_sym] = value
+      end
+
+      result
+    ensure
+      Rdkafka::Bindings.rd_kafka_conf_dump_free(dump_ptr, count) if dump_ptr
+      Rdkafka::Bindings.rd_kafka_conf_destroy(config) if config
+    end
+
     # Error that is returned by the underlying rdkafka error if an invalid configuration option is present.
     class ConfigError < RuntimeError; end
 
@@ -304,50 +338,66 @@ module Rdkafka
 
     # This method is only intended to be used to create a client,
     # using it in another way will leak memory.
+    #
+    # @param opaque [Object, nil] optional opaque pointer for callbacks
+    # @return [FFI::Pointer] native rdkafka configuration pointer
     def native_config(opaque = nil)
-      Rdkafka::Bindings.rd_kafka_conf_new.tap do |config|
-        # Create config
-        @config_hash.merge(REQUIRED_CONFIG).each do |key, value|
-          error_buffer = FFI::MemoryPointer.from_string(" " * 256)
-          result = Rdkafka::Bindings.rd_kafka_conf_set(
-            config,
-            key.to_s,
-            value.to_s,
-            error_buffer,
-            256
-          )
-          unless result == :config_ok
-            raise ConfigError.new(error_buffer.read_string)
-          end
+      config = Rdkafka::Bindings.rd_kafka_conf_new
+
+      # Create config
+      @config_hash.merge(REQUIRED_CONFIG).each do |key, value|
+        error_buffer = FFI::MemoryPointer.from_string(" " * 256)
+        result = Rdkafka::Bindings.rd_kafka_conf_set(
+          config,
+          key.to_s,
+          value.to_s,
+          error_buffer,
+          256
+        )
+        unless result == :config_ok
+          raise ConfigError.new(error_buffer.read_string)
         end
+      end
 
-        # Set opaque pointer that's used as a proxy for callbacks
-        if opaque
-          pointer = ::FFI::Pointer.new(:pointer, opaque.object_id)
-          Rdkafka::Bindings.rd_kafka_conf_set_opaque(config, pointer)
+      # Set opaque pointer that's used as a proxy for callbacks
+      if opaque
+        pointer = ::FFI::Pointer.new(:pointer, opaque.object_id)
+        Rdkafka::Bindings.rd_kafka_conf_set_opaque(config, pointer)
 
-          # Store opaque with the pointer as key. We use this approach instead
-          # of trying to convert the pointer to a Ruby object because there is
-          # no risk of a segfault this way.
-          Rdkafka::Config.opaques[pointer.to_i] = opaque
-        end
+        # Store opaque with the pointer as key. We use this approach instead
+        # of trying to convert the pointer to a Ruby object because there is
+        # no risk of a segfault this way.
+        Rdkafka::Config.opaques[pointer.to_i] = opaque
+      end
 
-        # Set log callback
-        Rdkafka::Bindings.rd_kafka_conf_set_log_cb(config, Rdkafka::Bindings::LogCallback)
+      # Set log callback
+      Rdkafka::Bindings.rd_kafka_conf_set_log_cb(config, Rdkafka::Bindings::LogCallback)
 
-        # Set stats callback
-        Rdkafka::Bindings.rd_kafka_conf_set_stats_cb(config, Rdkafka::Bindings::StatsCallback)
+      # Set stats callback
+      Rdkafka::Bindings.rd_kafka_conf_set_stats_cb(config, Rdkafka::Bindings::StatsCallback)
 
-        # Set error callback
-        Rdkafka::Bindings.rd_kafka_conf_set_error_cb(config, Rdkafka::Bindings::ErrorCallback)
+      # Set error callback
+      Rdkafka::Bindings.rd_kafka_conf_set_error_cb(config, Rdkafka::Bindings::ErrorCallback)
 
-        # Set oauth callback
-        if Rdkafka::Config.oauthbearer_token_refresh_callback
-          Rdkafka::Bindings.rd_kafka_conf_set_oauthbearer_token_refresh_cb(config, Rdkafka::Bindings::OAuthbearerTokenRefreshCallback)
-        end
+      # Set oauth callback
+      if Rdkafka::Config.oauthbearer_token_refresh_callback
+        Rdkafka::Bindings.rd_kafka_conf_set_oauthbearer_token_refresh_cb(config, Rdkafka::Bindings::OAuthbearerTokenRefreshCallback)
       end
+
+      config
+    rescue Exception
+      # rd_kafka_conf_new allocates a conf the caller owns until it is handed to rd_kafka_new. If
+      # we raise before returning it (e.g. an invalid option in the loop above), destroy it here
+      # so the native config is not leaked.
+      Rdkafka::Bindings.rd_kafka_conf_destroy(config) if config
+      raise
     end
 
+    # Creates a native Kafka handle
+    # @param config [FFI::Pointer] pointer to the native config
+    # @param type [Symbol] type of client (:rd_kafka_producer or :rd_kafka_consumer)
+    # @return [FFI::Pointer] pointer to the native Kafka handle
+    # @private
     def native_kafka(config, type)
       error_buffer = FFI::MemoryPointer.from_string(" " * 256)
       handle = Rdkafka::Bindings.rd_kafka_new(
@@ -358,14 +408,17 @@ module Rdkafka
       )
 
       if handle.null?
+        # On success rd_kafka_new takes ownership of the conf and frees it itself; on failure
+        # librdkafka keeps application ownership (see rdkafka.c), so we must destroy it here to
+        # avoid leaking the conf on every failed client creation.
+        Rdkafka::Bindings.rd_kafka_conf_destroy(config)
         raise ClientCreationError.new(error_buffer.read_string)
       end
 
       # Redirect log to handle's queue
-      Rdkafka::Bindings.rd_kafka_set_log_queue(
-        handle,
-        Rdkafka::Bindings.rd_kafka_queue_get_main(handle)
-      )
+      main_queue = Rdkafka::Bindings.rd_kafka_queue_get_main(handle)
+      Rdkafka::Bindings.rd_kafka_set_log_queue(handle, main_queue)
+      Rdkafka::Bindings.rd_kafka_queue_destroy(main_queue)
 
       # Return handle which should be closed using rd_kafka_destroy after usage.
       handle
@@ -377,10 +430,15 @@ module Rdkafka
     attr_accessor :producer
     attr_accessor :consumer_rebalance_listener
 
+    # Invokes the delivery callback on the producer if one is set
+    # @param delivery_report [Rdkafka::Producer::DeliveryReport] the delivery report
+    # @param delivery_handle [Rdkafka::Producer::DeliveryHandle] the delivery handle
     def call_delivery_callback(delivery_report, delivery_handle)
-      producer.call_delivery_callback(delivery_report, delivery_handle) if producer
+      producer&.call_delivery_callback(delivery_report, delivery_handle)
     end
 
+    # Invokes the on_partitions_assigned callback on the rebalance listener if set
+    # @param list [Rdkafka::Consumer::TopicPartitionList] the assigned partitions
     def call_on_partitions_assigned(list)
       return unless consumer_rebalance_listener
       return unless consumer_rebalance_listener.respond_to?(:on_partitions_assigned)
@@ -388,6 +446,8 @@ module Rdkafka
       consumer_rebalance_listener.on_partitions_assigned(list)
     end
 
+    # Invokes the on_partitions_revoked callback on the rebalance listener if set
+    # @param list [Rdkafka::Consumer::TopicPartitionList] the revoked partitions
     def call_on_partitions_revoked(list)
       return unless consumer_rebalance_listener
       return unless consumer_rebalance_listener.respond_to?(:on_partitions_revoked)

data/lib/rdkafka/consumer/headers.rb

@@ -4,19 +4,38 @@ module Rdkafka
   class Consumer
     # Interface to return headers for a consumer message
     module Headers
+      # Empty frozen hash used when there are no headers
       EMPTY_HEADERS = {}.freeze
 
+      # Key under which reusable scratch pointers are stored on the current thread/fiber
+      SCRATCH_KEY = :rdkafka_headers_scratch
+
+      private_constant :SCRATCH_KEY
+
       # Reads a librdkafka native message's headers and returns them as a Ruby Hash
       # where each key maps to either a String (single value) or Array<String> (multiple values)
       # to support duplicate headers per KIP-82
       #
       # @private
       #
-      # @param [Rdkafka::Bindings::Message] native_message
-      # @return [Hash<String, String|Array<String>>] headers Hash for the native_message
+      # @param native_message [Rdkafka::Bindings::Message] the native message to read headers from
+      # @return [Hash{String => String, Array<String>}] headers Hash for the native_message
       # @raise [Rdkafka::RdkafkaError] when fail to read headers
       def self.from_native(native_message)
-        headers_ptrptr = FFI::MemoryPointer.new(:pointer)
+        # Scratch output pointers for the header FFI calls, reused across messages. They are
+        # stored in fiber-local storage (`Thread.current[]` is fiber-local by design), so each
+        # thread and each fiber gets its own set and reuse is safe with fiber schedulers. They
+        # never escape this method and all data is read out of them before returning.
+        # Allocating them per message would cost several native allocations for every consumed
+        # message.
+        headers_ptrptr, name_ptrptr, value_ptrptr, size_ptr =
+          Thread.current[SCRATCH_KEY] ||= [
+            FFI::MemoryPointer.new(:pointer),
+            FFI::MemoryPointer.new(:pointer),
+            FFI::MemoryPointer.new(:pointer),
+            Rdkafka::Bindings::SizePtr.new
+          ].freeze
+
         err = Rdkafka::Bindings.rd_kafka_message_headers(native_message, headers_ptrptr)
 
         if err == Rdkafka::Bindings::RD_KAFKA_RESP_ERR__NOENT
@@ -27,10 +46,6 @@ module Rdkafka
 
         headers_ptr = headers_ptrptr.read_pointer
 
-        name_ptrptr = FFI::MemoryPointer.new(:pointer)
-        value_ptrptr = FFI::MemoryPointer.new(:pointer)
-        size_ptr = Rdkafka::Bindings::SizePtr.new
-
         headers = {}
 
         idx = 0

data/lib/rdkafka/consumer/message.rb

@@ -28,10 +28,11 @@ module Rdkafka
       # @return [Time, nil]
       attr_reader :timestamp
 
-      # @return [Hash<String, String>] a message headers
+      # @return [Hash{String => String}] message headers
       attr_reader :headers
 
       # @private
+      # @param native_message [Rdkafka::Bindings::Message] native message struct from librdkafka
       def initialize(native_message)
         # Set topic
         unless native_message[:rkt].null?
@@ -52,13 +53,11 @@ module Rdkafka
         # Set timestamp
         raw_timestamp = Rdkafka::Bindings.rd_kafka_message_timestamp(native_message, nil)
         @timestamp = if raw_timestamp && raw_timestamp > -1
-                       # Calculate seconds and microseconds
-                       seconds = raw_timestamp / 1000
-                       milliseconds = (raw_timestamp - seconds * 1000) * 1000
-                       Time.at(seconds, milliseconds)
-                     else
-                       nil
-                     end
+          # Calculate seconds and microseconds
+          seconds = raw_timestamp / 1000
+          milliseconds = (raw_timestamp - seconds * 1000) * 1000
+          Time.at(seconds, milliseconds)
+        end
 
         @headers = Headers.from_native(native_message)
       end
@@ -71,6 +70,11 @@ module Rdkafka
         "<Message in '#{topic}' with key '#{truncate(key)}', payload '#{truncate(payload)}', partition #{partition}, offset #{offset}, timestamp #{timestamp}#{is_headers}>"
       end
 
+      private
+
+      # Truncates a string for display purposes
+      # @param string [String, nil] the string to truncate
+      # @return [String, nil] truncated string or nil
       def truncate(string)
         if string && string.length > 40
           "#{string[0..39]}..."
@@ -78,9 +82,6 @@ module Rdkafka
           string
         end
       end
-
-      private
-
     end
   end
 end

data/lib/rdkafka/consumer/partition.rb

@@ -16,11 +16,20 @@ module Rdkafka
       # @return [Integer]
       attr_reader :err
 
+      # Partition metadata in the context of a consumer
+      # @return [String, nil]
+      attr_reader :metadata
+
       # @private
-      def initialize(partition, offset, err = 0)
+      # @param partition [Integer] partition number
+      # @param offset [Integer, nil] partition offset
+      # @param err [Integer] error code from librdkafka
+      # @param metadata [String, nil] partition metadata
+      def initialize(partition, offset, err = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR, metadata = nil)
         @partition = partition
         @offset = offset
         @err = err
+        @metadata = metadata
       end
 
       # Human readable representation of this partition.
@@ -28,7 +37,8 @@ module Rdkafka
       def to_s
         message = "<Partition #{partition}"
         message += " offset=#{offset}" if offset
-        message += " err=#{err}" if err != 0
+        message += " err=#{err}" if err != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+        message += " metadata=#{metadata}" if !metadata.nil?
         message += ">"
         message
       end
@@ -40,11 +50,12 @@ module Rdkafka
       end
 
       # Whether another partition is equal to this
+      # @param other [Object] object to compare with
       # @return [Boolean]
       def ==(other)
         self.class == other.class &&
-          self.partition == other.partition &&
-          self.offset == other.offset
+          partition == other.partition &&
+          offset == other.offset
       end
     end
   end

data/lib/rdkafka/consumer/topic_partition_list.rb

@@ -9,7 +9,7 @@ module Rdkafka
       # @param data [Hash{String => nil,Partition}] The topic and partition data or nil to create an empty list
       #
       # @return [TopicPartitionList]
-      def initialize(data=nil)
+      def initialize(data = nil)
         @data = data || {}
       end
 
@@ -18,10 +18,10 @@ module Rdkafka
       def count
         i = 0
         @data.each do |_topic, partitions|
-          if partitions
-            i += partitions.count
+          i += if partitions
+            partitions.count
           else
-            i+= 1
+            1
           end
         end
         i
@@ -49,15 +49,14 @@ module Rdkafka
       #
       # @example Add a topic with all topics up to a count
       #   tpl.add_topic("topic", 9)
-      #
-      def add_topic(topic, partitions=nil)
+      def add_topic(topic, partitions = nil)
         if partitions.nil?
           @data[topic.to_s] = nil
         else
           if partitions.is_a? Integer
             partitions = (0..partitions - 1)
           end
-          @data[topic.to_s] = partitions.map { |p| Partition.new(p, nil, 0) }
+          @data[topic.to_s] = partitions.map { |p| Partition.new(p, nil, Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR) }
         end
       end
 
@@ -65,11 +64,13 @@ module Rdkafka
       # Calling this method multiple times for the same topic will overwrite the previous configuraton.
       #
       # @param topic [String] The topic's name
-      # @param partitions_with_offsets [Hash<Integer, Integer>] The topic's partitions and offsets
-      #
+      # @param partitions_with_offsets [Hash{Integer => Integer}, Array<Consumer::Partition>] The topic's
+      #   partitions and offsets (Hash) or partitions with offsets and metadata (Array)
       # @return [nil]
       def add_topic_and_partitions_with_offsets(topic, partitions_with_offsets)
-        @data[topic.to_s] = partitions_with_offsets.map { |p, o| Partition.new(p, o) }
+        @data[topic.to_s] = partitions_with_offsets.map do |p, o|
+          p.is_a?(Partition) ? p : Partition.new(p, o)
+        end
       end
 
       # Return a `Hash` with the topics as keys and and an array of partition information as the value if present.
@@ -85,8 +86,11 @@ module Rdkafka
         "<TopicPartitionList: #{to_h}>"
       end
 
+      # Check equality with another TopicPartitionList
+      # @param other [TopicPartitionList] object to compare with
+      # @return [Boolean]
       def ==(other)
-        self.to_h == other.to_h
+        to_h == other.to_h
       end
 
       # Create a new topic partition list based of a native one.
@@ -105,16 +109,21 @@ module Rdkafka
         native_tpl[:cnt].times do |i|
           ptr = native_tpl[:elems] + (i * Rdkafka::Bindings::TopicPartition.size)
           elem = Rdkafka::Bindings::TopicPartition.new(ptr)
-          if elem[:partition] == -1
+          if elem[:partition] == Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
             data[elem[:topic]] = nil
           else
             partitions = data[elem[:topic]] || []
             offset = if elem[:offset] == Rdkafka::Bindings::RD_KAFKA_OFFSET_INVALID
-                       nil
-                     else
-                       elem[:offset]
-                     end
-            partition = Partition.new(elem[:partition], offset, elem[:err])
+              nil
+            else
+              elem[:offset]
+            end
+            partition = Partition.new(
+              elem[:partition],
+              offset,
+              elem[:err],
+              elem[:metadata].null? ? nil : elem[:metadata].read_string(elem[:metadata_size])
+            )
             partitions.push(partition)
             data[elem[:topic]] = partitions
           end
@@ -133,33 +142,50 @@ module Rdkafka
       def to_native_tpl
         tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(count)
 
-        @data.each do |topic, partitions|
-          if partitions
-            partitions.each do |p|
-              Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
-                tpl,
-                topic,
-                p.partition
-              )
-
-              if p.offset
-                offset = p.offset.is_a?(Time) ? p.offset.to_f * 1_000 : p.offset
-
-                Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(
+        begin
+          @data.each do |topic, partitions|
+            if partitions
+              partitions.each do |p|
+                ref = Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
                   tpl,
                   topic,
-                  p.partition,
-                  offset
+                  p.partition
                 )
+
+                if p.metadata
+                  part = Rdkafka::Bindings::TopicPartition.new(ref)
+                  str_ptr = FFI::MemoryPointer.from_string(p.metadata)
+                  # The metadata string is owned by librdkafka once handed over and released here:
+                  # https://github.com/confluentinc/librdkafka/blob/e03d3bb91ed92a38f38d9806b8d8deffe78a1de5/src/rdkafka_partition.c#L2682C18-L2682C18
+                  str_ptr.autorelease = false
+                  part[:metadata] = str_ptr
+                  part[:metadata_size] = p.metadata.bytesize
+                end
+
+                if p.offset
+                  offset = p.offset.is_a?(Time) ? p.offset.to_f * 1_000 : p.offset
+
+                  Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(
+                    tpl,
+                    topic,
+                    p.partition,
+                    offset
+                  )
+                end
               end
+            else
+              Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
+                tpl,
+                topic,
+                Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
+              )
             end
-          else
-            Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
-              tpl,
-              topic,
-              -1
-            )
           end
+        rescue Exception
+          # The native list is caller-owned until it is handed to librdkafka. If population raises
+          # partway (e.g. an invalid partition), destroy it here so it is not leaked.
+          Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
+          raise
         end
 
         tpl