karafka-rdkafka 0.27.2-aarch64-linux-gnu → 0.28.1-aarch64-linux-gnu

data/lib/rdkafka/consumer/headers.rb

@@ -7,6 +7,11 @@ module Rdkafka
       # Empty frozen hash used when there are no headers
       EMPTY_HEADERS = {}.freeze
 
+      # Key under which reusable scratch pointers are stored on the current thread/fiber
+      SCRATCH_KEY = :rdkafka_headers_scratch
+
+      private_constant :SCRATCH_KEY
+
       # Reads a librdkafka native message's headers and returns them as a Ruby Hash
       # where each key maps to either a String (single value) or Array<String> (multiple values)
       # to support duplicate headers per KIP-82
@@ -17,7 +22,20 @@ module Rdkafka
       # @return [Hash{String => String, Array<String>}] headers Hash for the native_message
       # @raise [Rdkafka::RdkafkaError] when fail to read headers
       def self.from_native(native_message)
-        headers_ptrptr = FFI::MemoryPointer.new(:pointer)
+        # Scratch output pointers for the header FFI calls, reused across messages. They are
+        # stored in fiber-local storage (`Thread.current[]` is fiber-local by design), so each
+        # thread and each fiber gets its own set and reuse is safe with fiber schedulers. They
+        # never escape this method and all data is read out of them before returning.
+        # Allocating them per message would cost several native allocations for every consumed
+        # message.
+        headers_ptrptr, name_ptrptr, value_ptrptr, size_ptr =
+          Thread.current[SCRATCH_KEY] ||= [
+            FFI::MemoryPointer.new(:pointer),
+            FFI::MemoryPointer.new(:pointer),
+            FFI::MemoryPointer.new(:pointer),
+            Rdkafka::Bindings::SizePtr.new
+          ].freeze
+
         err = Rdkafka::Bindings.rd_kafka_message_headers(native_message, headers_ptrptr)
 
         if err == Rdkafka::Bindings::RD_KAFKA_RESP_ERR__NOENT
@@ -28,10 +46,6 @@ module Rdkafka
 
         headers_ptr = headers_ptrptr.read_pointer
 
-        name_ptrptr = FFI::MemoryPointer.new(:pointer)
-        value_ptrptr = FFI::MemoryPointer.new(:pointer)
-        size_ptr = Rdkafka::Bindings::SizePtr.new
-
         headers = {}
 
         idx = 0

data/lib/rdkafka/consumer/topic_partition_list.rb

@@ -6,7 +6,8 @@ module Rdkafka
     class TopicPartitionList
       # Create a topic partition list.
       #
-      # @param data [Hash{String => nil,Partition}] The topic and partition data or nil to create an empty list
+      # @param data [Hash{String => nil,Partition}] The topic and partition data or nil to create an
+      #   empty list
       #
       # @return [TopicPartitionList]
       def initialize(data = nil)
@@ -34,10 +35,12 @@ module Rdkafka
       end
 
       # Add a topic with optionally partitions to the list.
-      # Calling this method multiple times for the same topic will overwrite the previous configuraton.
+      # Calling this method multiple times for the same topic will overwrite the previous
+      # configuraton.
       #
       # @param topic [String] The topic's name
-      # @param partitions [Array<Integer>, Range<Integer>, Integer] The topic's partitions or partition count
+      # @param partitions [Array<Integer>, Range<Integer>, Integer] The topic's partitions or
+      #   partition count
       #
       # @return [nil]
       #
@@ -61,11 +64,12 @@ module Rdkafka
       end
 
       # Add a topic with partitions and offsets set to the list
-      # Calling this method multiple times for the same topic will overwrite the previous configuraton.
+      # Calling this method multiple times for the same topic will overwrite the previous
+      # configuraton.
       #
       # @param topic [String] The topic's name
-      # @param partitions_with_offsets [Hash{Integer => Integer}, Array<Consumer::Partition>] The topic's
-      #   partitions and offsets (Hash) or partitions with offsets and metadata (Array)
+      # @param partitions_with_offsets [Hash{Integer => Integer}, Array<Consumer::Partition>] The
+      #   topic's partitions and offsets (Hash) or partitions with offsets and metadata (Array)
       # @return [nil]
       def add_topic_and_partitions_with_offsets(topic, partitions_with_offsets)
         @data[topic.to_s] = partitions_with_offsets.map do |p, o|
@@ -73,7 +77,8 @@ module Rdkafka
         end
       end
 
-      # Return a `Hash` with the topics as keys and and an array of partition information as the value if present.
+      # Return a `Hash` with the topics as keys and and an array of partition information as the
+      # value if present.
       #
       # @return [Hash{String => Array<Partition>,nil}]
       def to_h
@@ -97,7 +102,8 @@ module Rdkafka
       #
       # @private
       #
-      # @param pointer [FFI::Pointer] Optional pointer to an existing native list. Its contents will be copied.
+      # @param pointer [FFI::Pointer] Optional pointer to an existing native list. Its contents will
+      #   be copied.
       #
       # @return [TopicPartitionList]
       def self.from_native_tpl(pointer)
@@ -118,7 +124,6 @@ module Rdkafka
             else
               elem[:offset]
             end
-
             partition = Partition.new(
               elem[:partition],
               offset,
@@ -136,51 +141,61 @@ module Rdkafka
 
       # Create a native tpl with the contents of this object added.
       #
-      # The pointer will be cleaned by `rd_kafka_topic_partition_list_destroy` when GC releases it.
+      # The returned pointer is caller-owned and must be released with
+      # `rd_kafka_topic_partition_list_destroy`. It is not cleaned up by GC.
       #
       # @private
       # @return [FFI::Pointer]
       def to_native_tpl
         tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(count)
 
-        @data.each do |topic, partitions|
-          if partitions
-            partitions.each do |p|
-              ref = Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
-                tpl,
-                topic,
-                p.partition
-              )
-
-              # Remove the respond to check after karafka 2.3.0 is released
-              if p.respond_to?(:metadata) && p.metadata
-                part = Rdkafka::Bindings::TopicPartition.new(ref)
-                str_ptr = FFI::MemoryPointer.from_string(p.metadata)
-                # released here:
-                # https://github.com/confluentinc/librdkafka/blob/e03d3bb91ed92a38f38d9806b8d8deffe78a1de5/src/rdkafka_partition.c#L2682C18-L2682C18
-                str_ptr.autorelease = false
-                part[:metadata] = str_ptr
-                part[:metadata_size] = p.metadata.bytesize
-              end
-
-              if p.offset
-                offset = p.offset.is_a?(Time) ? p.offset.to_f * 1_000 : p.offset
-
-                Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(
+        begin
+          @data.each do |topic, partitions|
+            if partitions
+              partitions.each do |p|
+                ref = Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
                   tpl,
                   topic,
-                  p.partition,
-                  offset
+                  p.partition
                 )
+
+                # Remove the respond to check after karafka 2.3.0 is released
+                if p.respond_to?(:metadata) && p.metadata
+                  part = Rdkafka::Bindings::TopicPartition.new(ref)
+                  str_ptr = FFI::MemoryPointer.from_string(p.metadata)
+                  # released here:
+                  # https://github.com/confluentinc/librdkafka/blob/e03d3bb91ed92a38f38d9806b8d8deffe78a1de5/src/rdkafka_partition.c#L2682C18-L2682C18
+                  str_ptr.autorelease = false
+                  part[:metadata] = str_ptr
+                  part[:metadata_size] = p.metadata.bytesize
+                end
+
+                if p.offset
+                  offset = p.offset.is_a?(Time) ? p.offset.to_f * 1_000 : p.offset
+
+                  Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(
+                    tpl,
+                    topic,
+                    p.partition,
+                    offset
+                  )
+                end
               end
+            else
+              Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
+                tpl,
+                topic,
+                Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
+              )
             end
-          else
-            Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
-              tpl,
-              topic,
-              Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
-            )
           end
+        rescue Exception
+          # The native list is caller-owned and only handed back on success. If population raises
+          # partway (e.g. non-string metadata, or an offset FFI can't coerce to int64) we must
+          # destroy it here, otherwise the half-built list leaks.
+          Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
+
+          raise
         end
 
         tpl

data/lib/rdkafka/consumer.rb

@@ -14,14 +14,48 @@ module Rdkafka
     include Enumerable
     include Helpers::Time
     include Helpers::OAuth
+    include Helpers::Metadata
+    include Helpers::ListOffsets
 
     # @private
     # @param native_kafka [NativeKafka] wrapper around the native Kafka consumer handle
     def initialize(native_kafka)
       @native_kafka = native_kafka
+      # Single-element holder shared with the GC finalizer so it can destroy the lazily created
+      # consumer queue without capturing `self` (capturing the consumer in its own finalizer would
+      # pin it and prevent it from ever being collected).
+      @consumer_queue_holder = []
+
+      # Makes sure the consumer is closed (consumer queue destroyed and native client destroyed)
+      # before it gets GCed by Ruby.
+      ObjectSpace.define_finalizer(self, self.class.finalizer(native_kafka, @consumer_queue_holder))
+    end
+
+    # Builds the GC finalizer for a consumer. It mirrors {#close}: close the consumer, destroy the
+    # consumer-queue reference, then destroy the native client. The default `NativeKafka#finalizer`
+    # went straight to `rd_kafka_destroy`, leaving the consumer-queue reference (from
+    # `rd_kafka_queue_get_consumer`, taken by `poll_batch`) dangling - which can make
+    # `rd_kafka_destroy` block inside the finalizer (process hang at GC/shutdown) or leak the
+    # handle.
+    #
+    # @private
+    # @param native_kafka [NativeKafka] the wrapped native client
+    # @param queue_holder [Array] single-element holder carrying the consumer queue pointer
+    #   (or empty)
+    # @return [Proc] finalizer proc that must not reference the consumer instance
+    def self.finalizer(native_kafka, queue_holder)
+      proc do
+        next if native_kafka.closed?
+
+        native_kafka.synchronize do |inner|
+          Rdkafka::Bindings.rd_kafka_consumer_close(inner)
+
+          queue = queue_holder[0]
+          Rdkafka::Bindings.rd_kafka_queue_destroy(queue) if queue
+        end
 
-      # Makes sure, that native kafka gets closed before it gets GCed by Ruby
-      ObjectSpace.define_finalizer(self, native_kafka.finalizer)
+        native_kafka.close
+      end
     end
 
     # Starts the native Kafka polling thread and kicks off the init polling
@@ -93,11 +127,6 @@ module Rdkafka
     # @return [nil]
     # @raise [Rdkafka::ClosedConsumerError] if called on a closed consumer
     #
-    # @note This method holds the inner lock until the queue is empty or `:stop` is returned.
-    #   Other consumer operations will wait until this method returns.
-    # @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
-    # @note Do NOT use this if `consumer_poll_set` was set to `true`
-    #
     # @example Drain all pending events
     #   consumer.events_poll_nb_each { |_count| }
     #
@@ -106,6 +135,10 @@ module Rdkafka
     #   consumer.events_poll_nb_each do |_count|
     #     :stop if monotonic_now >= deadline
     #   end
+    # @note This method holds the inner lock until the queue is empty or `:stop` is returned.
+    #   Other consumer operations will wait until this method returns.
+    # @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
+    # @note Do NOT use this if `consumer_poll_set` was set to `true`
     def events_poll_nb_each
       closed_consumer_check(__method__)
 
@@ -131,14 +164,6 @@ module Rdkafka
     # @raise [Rdkafka::ClosedConsumerError] if called on a closed consumer
     # @raise [Rdkafka::RdkafkaError] if a Kafka error occurs while polling
     #
-    # @note This method uses `rd_kafka_consumer_poll` to fetch messages, unlike
-    #   `events_poll_nb_each` which uses `rd_kafka_poll` for event callbacks (delivery reports,
-    #   statistics, etc.). For consumers, use this method to receive messages and
-    #   `events_poll_nb_each` for processing background events.
-    # @note This method holds the inner lock for the duration. Other consumer operations
-    #   will wait until this method returns.
-    # @note Timeout/max_messages logic should be implemented by the caller
-    #
     # @example Process messages until queue is empty
     #   consumer.poll_nb_each do |message|
     #     process(message)
@@ -151,6 +176,13 @@ module Rdkafka
     #     count += 1
     #     :stop if count >= 10
     #   end
+    # @note This method uses `rd_kafka_consumer_poll` to fetch messages, unlike
+    #   `events_poll_nb_each` which uses `rd_kafka_poll` for event callbacks (delivery reports,
+    #   statistics, etc.). For consumers, use this method to receive messages and
+    #   `events_poll_nb_each` for processing background events.
+    # @note This method holds the inner lock for the duration. Other consumer operations
+    #   will wait until this method returns.
+    # @note Timeout/max_messages logic should be implemented by the caller
     def poll_nb_each
       closed_consumer_check(__method__)
 
@@ -187,6 +219,7 @@ module Rdkafka
         if @consumer_queue
           Rdkafka::Bindings.rd_kafka_queue_destroy(@consumer_queue)
           @consumer_queue = nil
+          @consumer_queue_holder[0] = nil
         end
       end
 
@@ -404,14 +437,18 @@ module Rdkafka
     end
 
     # Return the current positions (offsets) for topics and partitions.
-    # The offset field of each requested partition will be set to the offset of the last consumed message + 1, or nil in case there was no previous message.
+    # The offset field of each requested partition will be set to the offset of the last consumed
+    # message + 1, or nil in case there was no previous message.
     #
-    # @param list [TopicPartitionList, nil] The topic with partitions to get the offsets for or nil to use the current subscription.
+    # @param list [TopicPartitionList, nil] The topic with partitions to get the offsets for or nil
+    #   to use the current subscription.
     #
     # @return [TopicPartitionList]
     #
     # @raise [RdkafkaError] When getting the positions fails.
     def position(list = nil)
+      closed_consumer_check(__method__)
+
       if list.nil?
         list = assignment
       elsif !list.is_a?(TopicPartitionList)
@@ -466,39 +503,77 @@ module Rdkafka
     # possible to create one yourself, in this case you have to provide a list that
     # already contains all the partitions you need the lag for.
     #
+    # The end offsets of all requested partitions are fetched in a single batched
+    # {#list_offsets} query - librdkafka fans it out to the involved partition leaders
+    # internally and concurrently - instead of one blocking {#query_watermark_offsets}
+    # broker roundtrip per partition.
+    #
     # @param topic_partition_list [TopicPartitionList] The list to calculate lag for.
-    # @param watermark_timeout_ms [Integer] The timeout for each query watermark call.
+    # @param watermark_timeout_ms [Integer] The timeout for the batched end-offsets query.
     # @return [Hash{String => Hash{Integer => Integer}}] A hash containing all topics with the lag
     #   per partition
+    # @raise [ClosedConsumerError] when the consumer is closed
     # @raise [RdkafkaError] When querying the broker fails.
     def lag(topic_partition_list, watermark_timeout_ms = Defaults::CONSUMER_LAG_TIMEOUT_MS)
+      closed_consumer_check(__method__)
+
       out = {}
+      request = {}
+      partitions_by_topic = topic_partition_list.to_h
+
+      partitions_by_topic.each do |topic, partitions|
+        out[topic] = {}
+
+        partitions.each do |p|
+          next if p.offset.nil?
+
+          (request[topic] ||= []) << { partition: p.partition, offset: :latest }
+        end
+      end
+
+      return out if request.empty?
+
+      report = begin
+        # The isolation level is forwarded so the end offsets match what the old per-partition
+        # watermark query returned: librdkafka resolves that query with the consumer's configured
+        # `isolation.level` (LSO for the default read_committed), while the admin-style batched
+        # query would otherwise default to read_uncommitted (true high watermark).
+        list_offsets(request, isolation_level: isolation_level)
+          .wait(max_wait_timeout_ms: watermark_timeout_ms)
+      rescue AbstractHandle::WaitTimeoutError
+        # Keep the pre-batching contract: a slow broker surfaced as a timed-out RdkafkaError
+        # from the per-partition watermark query, not as a handle wait timeout.
+        raise RdkafkaError.new(
+          Rdkafka::Bindings::RD_KAFKA_RESP_ERR__TIMED_OUT,
+          "Error querying watermark offsets of '#{request.keys.join(", ")}'"
+        )
+      end
+
+      end_offsets = {}
+      report.offsets.each do |result|
+        (end_offsets[result[:topic]] ||= {})[result[:partition]] = result[:offset]
+      end
 
-      topic_partition_list.to_h.each do |topic, partitions|
-        # Query high watermarks for this topic's partitions
-        # and compare to the offset in the list.
-        topic_out = {}
+      partitions_by_topic.each do |topic, partitions|
         partitions.each do |p|
           next if p.offset.nil?
-          _low, high = query_watermark_offsets(
-            topic,
-            p.partition,
-            watermark_timeout_ms
-          )
-          topic_out[p.partition] = high - p.offset
+
+          out[topic][p.partition] = end_offsets.fetch(topic).fetch(p.partition) - p.offset
         end
-        out[topic] = topic_out
       end
+
       out
     end
 
     # Returns the ClusterId as reported in broker metadata.
     #
-    # @return [String, nil]
-    def cluster_id
+    # @param timeout_ms [Integer] how long to block waiting for metadata when the cluster id is
+    #   not already cached (0 for a non-blocking call)
+    # @return [String, nil] the cluster id, or nil if it could not be retrieved in time
+    def cluster_id(timeout_ms = Defaults::CONSUMER_CLUSTER_ID_TIMEOUT_MS)
       closed_consumer_check(__method__)
       @native_kafka.with_inner do |inner|
-        Rdkafka::Bindings.rd_kafka_clusterid(inner)
+        read_and_free_native_string(inner, Rdkafka::Bindings.rd_kafka_clusterid(inner, timeout_ms))
       end
     end
 
@@ -510,7 +585,7 @@ module Rdkafka
     def member_id
       closed_consumer_check(__method__)
       @native_kafka.with_inner do |inner|
-        Rdkafka::Bindings.rd_kafka_memberid(inner)
+        read_and_free_native_string(inner, Rdkafka::Bindings.rd_kafka_memberid(inner))
       end
     end
 
@@ -519,7 +594,7 @@ module Rdkafka
     # When using this `enable.auto.offset.store` should be set to `false` in the config.
     #
     # @param message [Rdkafka::Consumer::Message] The message which offset will be stored
-    # @param metadata [String, nil] commit metadata string or nil if none
+    # @param metadata [String, nil] commit metadata string to store alongside the offset
     # @return [nil]
     # @raise [RdkafkaError] When storing the offset fails
     def store_offset(message, metadata = nil)
@@ -573,9 +648,8 @@ module Rdkafka
       seek_by(message.topic, message.partition, message.offset)
     end
 
-    # Seek to a particular message by providing the topic, partition and offset.
-    # The next poll on the topic/partition will return the
-    # message at the given offset.
+    # Seek to a particular message by providing the topic, partition and offset. The next poll on
+    # the topic/partition will return the message at the given offset.
     #
     # @param topic [String] The topic in which to seek
     # @param partition [Integer] The partition number to seek
@@ -802,7 +876,7 @@ module Rdkafka
     # returns without further waiting.
     #
     # Error events (e.g. `:partition_eof`) are returned inline as {RdkafkaError} objects
-    # rather than raised, so callers receive the complete batch — both messages and errors —
+    # rather than raised, so callers receive the complete batch - both messages and errors -
     # and can decide how to handle each. This is particularly useful when multiple partitions
     # signal EOF simultaneously: all signals appear in the returned array rather than only
     # the first one being raised and the rest silently discarded.
@@ -847,8 +921,17 @@ module Rdkafka
             next
           end
 
-          results << Rdkafka::Consumer::Message.new(native_message)
-          Rdkafka::Bindings.rd_kafka_message_destroy(ptr)
+          begin
+            results << Rdkafka::Consumer::Message.new(native_message)
+          rescue Rdkafka::RdkafkaError => e
+            # A message that fails to build (e.g. a header read error) is surfaced inline as an
+            # error event rather than discarding the whole batch - including the messages already
+            # built - and raising, which silently lost them once their offsets had been stored.
+            results << e
+          ensure
+            Rdkafka::Bindings.rd_kafka_message_destroy(ptr)
+          end
+
           i += 1
         end
       ensure
@@ -868,15 +951,15 @@ module Rdkafka
     # particularly useful in fiber scheduler contexts where GVL release/reacquire
     # overhead is wasteful since we don't expect to wait.
     #
+    # @param timeout_ms [Integer] Timeout waiting for the first message
+    #   (default: 0 for non-blocking)
+    # @param max_items [Integer] Maximum number of messages to return per call
+    # @return [Array<Message, RdkafkaError>] Batch of messages and/or error events in arrival order
+    # @raise [ClosedConsumerError] When called on a closed consumer
     # @note Since the GVL is not released, a non-zero timeout_ms will block all Ruby
     #   threads/fibers for the duration. Use {#poll_batch} if you need a blocking wait.
     #
     # Error events are returned inline as {RdkafkaError} objects; see {#poll_batch} for details.
-    #
-    # @param timeout_ms [Integer] Timeout waiting for the first message (default: 0 for non-blocking)
-    # @param max_items [Integer] Maximum number of messages to return per call
-    # @return [Array<Message, RdkafkaError>] Batch of messages and/or error events in arrival order
-    # @raise [ClosedConsumerError] When called on a closed consumer
     def poll_batch_nb(timeout_ms = 0, max_items: 100)
       closed_consumer_check(__method__)
 
@@ -913,8 +996,17 @@ module Rdkafka
             next
           end
 
-          results << Rdkafka::Consumer::Message.new(native_message)
-          Rdkafka::Bindings.rd_kafka_message_destroy(ptr)
+          begin
+            results << Rdkafka::Consumer::Message.new(native_message)
+          rescue Rdkafka::RdkafkaError => e
+            # A message that fails to build (e.g. a header read error) is surfaced inline as an
+            # error event rather than discarding the whole batch - including the messages already
+            # built - and raising, which silently lost them once their offsets had been stored.
+            results << e
+          ensure
+            Rdkafka::Bindings.rd_kafka_message_destroy(ptr)
+          end
+
           i += 1
         end
       ensure
@@ -1000,12 +1092,62 @@ module Rdkafka
     def closed_consumer_check(method)
       raise Rdkafka::ClosedConsumerError.new(method) if closed?
     end
+    alias_method :closed_check, :closed_consumer_check
+
+    # Reads a librdkafka-allocated string and frees the underlying native buffer.
+    #
+    # `rd_kafka_memberid`/`rd_kafka_clusterid` return a string the caller owns and must release
+    # with `rd_kafka_mem_free`; without this the buffer leaks on every call.
+    #
+    # @param inner [FFI::Pointer] the native client handle (needed to free the buffer)
+    # @param ptr [FFI::Pointer] the librdkafka-allocated string, possibly NULL
+    # @return [String, nil] the copied string, or nil when the pointer was NULL
+    def read_and_free_native_string(inner, ptr)
+      return nil if ptr.null?
+
+      ptr.read_string
+    ensure
+      Rdkafka::Bindings.rd_kafka_mem_free(inner, ptr) unless ptr.null?
+    end
+
+    # Reads this consumer's effective `isolation.level` from the live librdkafka configuration
+    # and maps it to the numeric isolation level constant. Memoized: the value cannot change
+    # after client creation.
+    #
+    # @return [Integer] `RD_KAFKA_ISOLATION_LEVEL_READ_COMMITTED` or
+    #   `RD_KAFKA_ISOLATION_LEVEL_READ_UNCOMMITTED`
+    # @raise [Rdkafka::Config::ConfigError] when the property cannot be read
+    def isolation_level
+      @isolation_level ||= @native_kafka.with_inner do |inner|
+        conf = Rdkafka::Bindings.rd_kafka_conf(inner)
+
+        size_ptr = Rdkafka::Bindings::SizePtr.new
+        size_ptr[:value] = 64
+        value_ptr = FFI::MemoryPointer.new(:char, 64)
+
+        result = Rdkafka::Bindings.rd_kafka_conf_get(conf, "isolation.level", value_ptr, size_ptr)
+
+        if result != :config_ok
+          raise Rdkafka::Config::ConfigError.new("Could not read isolation.level: #{result}")
+        end
+
+        if value_ptr.read_string == "read_committed"
+          Rdkafka::Bindings::RD_KAFKA_ISOLATION_LEVEL_READ_COMMITTED
+        else
+          Rdkafka::Bindings::RD_KAFKA_ISOLATION_LEVEL_READ_UNCOMMITTED
+        end
+      end
+    end
 
     # Returns the consumer queue pointer, lazily initialized
     # @return [FFI::Pointer] consumer queue handle
     def consumer_queue
       @consumer_queue ||= @native_kafka.with_inner do |inner|
-        Rdkafka::Bindings.rd_kafka_queue_get_consumer(inner)
+        queue = Rdkafka::Bindings.rd_kafka_queue_get_consumer(inner)
+        # Share the pointer with the finalizer so it is destroyed even if the consumer is GC'd
+        # without an explicit close.
+        @consumer_queue_holder[0] = queue
+        queue
       end
     end
 

data/lib/rdkafka/defaults.rb

@@ -3,8 +3,17 @@
 module Rdkafka
   # Provides default timeout and configuration values used throughout the library.
   #
-  # These constants standardize timing values across consumers, producers, and admin clients.
-  # Values are specified in milliseconds (ms) unless otherwise noted.
+  # All timeout values can be overridden per-call via method parameters. These constants provide
+  # a central place to understand and reference the default values used across the library.
+  #
+  # @example Overriding a timeout per-call
+  #   consumer.committed(timeout_ms: 5_000)  # Use 5 seconds instead of default 2 seconds
+  #
+  # @example Checking the default value
+  #   Rdkafka::Defaults::CONSUMER_COMMITTED_TIMEOUT_MS  # => 2000
+  # @note These are rdkafka-ruby defaults, not librdkafka configuration options.
+  #   For librdkafka options, see:
+  #   https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md
   module Defaults
     # Consumer timeout for fetching committed offsets
     # @see Consumer#committed
@@ -22,6 +31,10 @@ module Rdkafka
     # @see Consumer#offsets_for_times
     CONSUMER_OFFSETS_FOR_TIMES_TIMEOUT_MS = 1_000
 
+    # Consumer timeout for fetching the cluster id when it is not already cached from metadata
+    # @see Consumer#cluster_id
+    CONSUMER_CLUSTER_ID_TIMEOUT_MS = 1_000
+
     # Consumer timeout for poll operations (used in each iteration)
     # @see Consumer#each
     CONSUMER_POLL_TIMEOUT_MS = 250
@@ -55,14 +68,30 @@ module Rdkafka
     # @see Admin#metadata
     METADATA_TIMEOUT_MS = 2_000
 
-    # Maximum retries for metadata requests on transient errors
+    # Hard ceiling on metadata fetch attempts on transient errors (backstop; the retry budget
+    # normally ends the loop first)
     # @see Metadata#initialize
     METADATA_MAX_RETRIES = 10
 
+    # Minimum metadata fetch attempts before the retry budget may end the loop, so a slow broker
+    # (whose requests each consume the full timeout) still gets a few tries
+    # @see Metadata#initialize
+    METADATA_MIN_ATTEMPTS = 3
+
+    # Soft wall-clock budget for the whole metadata retry loop; past it (and past
+    # METADATA_MIN_ATTEMPTS) the loop stops so a synchronous fetch cannot block the caller for long
+    # @see Metadata#initialize
+    METADATA_RETRY_BUDGET_MS = 5_000
+
     # Base backoff time for metadata retry (100ms = 0.1s)
     # @see Metadata#initialize
     METADATA_RETRY_BACKOFF_BASE_MS = 100
 
+    # Maximum backoff time between metadata retries. Caps the exponential backoff so a long retry
+    # sequence against an unhealthy cluster cannot block the calling thread for minutes.
+    # @see Metadata#initialize
+    METADATA_RETRY_BACKOFF_MAX_MS = 1_000
+
     # Default wait timeout for operation handles
     # @see AbstractHandle#wait
     HANDLE_WAIT_TIMEOUT_MS = 60_000