rdkafka 0.28.0-aarch64-linux-gnu → 0.29.1-aarch64-linux-gnu

data/lib/rdkafka/consumer/topic_partition_list.rb

@@ -6,7 +6,8 @@ module Rdkafka
     class TopicPartitionList
       # Create a topic partition list.
       #
-      # @param data [Hash{String => nil,Partition}] The topic and partition data or nil to create an empty list
+      # @param data [Hash{String => nil,Partition}] The topic and partition data or nil to create an
+      #   empty list
       #
       # @return [TopicPartitionList]
       def initialize(data = nil)
@@ -34,10 +35,12 @@ module Rdkafka
       end
 
       # Add a topic with optionally partitions to the list.
-      # Calling this method multiple times for the same topic will overwrite the previous configuraton.
+      # Calling this method multiple times for the same topic will overwrite the previous
+      # configuraton.
       #
       # @param topic [String] The topic's name
-      # @param partitions [Array<Integer>, Range<Integer>, Integer] The topic's partitions or partition count
+      # @param partitions [Array<Integer>, Range<Integer>, Integer] The topic's partitions or
+      #   partition count
       #
       # @return [nil]
       #
@@ -61,11 +64,12 @@ module Rdkafka
       end
 
       # Add a topic with partitions and offsets set to the list
-      # Calling this method multiple times for the same topic will overwrite the previous configuraton.
+      # Calling this method multiple times for the same topic will overwrite the previous
+      # configuraton.
       #
       # @param topic [String] The topic's name
-      # @param partitions_with_offsets [Hash{Integer => Integer}, Array<Consumer::Partition>] The topic's
-      #   partitions and offsets (Hash) or partitions with offsets and metadata (Array)
+      # @param partitions_with_offsets [Hash{Integer => Integer}, Array<Consumer::Partition>] The
+      #   topic's partitions and offsets (Hash) or partitions with offsets and metadata (Array)
       # @return [nil]
       def add_topic_and_partitions_with_offsets(topic, partitions_with_offsets)
         @data[topic.to_s] = partitions_with_offsets.map do |p, o|
@@ -73,7 +77,8 @@ module Rdkafka
         end
       end
 
-      # Return a `Hash` with the topics as keys and and an array of partition information as the value if present.
+      # Return a `Hash` with the topics as keys and and an array of partition information as the
+      # value if present.
       #
       # @return [Hash{String => Array<Partition>,nil}]
       def to_h
@@ -97,7 +102,8 @@ module Rdkafka
       #
       # @private
       #
-      # @param pointer [FFI::Pointer] Optional pointer to an existing native list. Its contents will be copied.
+      # @param pointer [FFI::Pointer] Optional pointer to an existing native list. Its contents will
+      #   be copied.
       #
       # @return [TopicPartitionList]
       def self.from_native_tpl(pointer)
@@ -118,7 +124,12 @@ module Rdkafka
             else
               elem[:offset]
             end
-            partition = Partition.new(elem[:partition], offset, elem[:err])
+            partition = Partition.new(
+              elem[:partition],
+              offset,
+              elem[:err],
+              elem[:metadata].null? ? nil : elem[:metadata].read_string(elem[:metadata_size])
+            )
             partitions.push(partition)
             data[elem[:topic]] = partitions
           end
@@ -137,33 +148,50 @@ module Rdkafka
       def to_native_tpl
         tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(count)
 
-        @data.each do |topic, partitions|
-          if partitions
-            partitions.each do |p|
-              Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
-                tpl,
-                topic,
-                p.partition
-              )
-
-              if p.offset
-                offset = p.offset.is_a?(Time) ? p.offset.to_f * 1_000 : p.offset
-
-                Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(
+        begin
+          @data.each do |topic, partitions|
+            if partitions
+              partitions.each do |p|
+                ref = Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
                   tpl,
                   topic,
-                  p.partition,
-                  offset
+                  p.partition
                 )
+
+                if p.metadata
+                  part = Rdkafka::Bindings::TopicPartition.new(ref)
+                  str_ptr = FFI::MemoryPointer.from_string(p.metadata)
+                  # The metadata string is owned by librdkafka once handed over and released here:
+                  # https://github.com/confluentinc/librdkafka/blob/e03d3bb91ed92a38f38d9806b8d8deffe78a1de5/src/rdkafka_partition.c#L2682C18-L2682C18
+                  str_ptr.autorelease = false
+                  part[:metadata] = str_ptr
+                  part[:metadata_size] = p.metadata.bytesize
+                end
+
+                if p.offset
+                  offset = p.offset.is_a?(Time) ? p.offset.to_f * 1_000 : p.offset
+
+                  Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(
+                    tpl,
+                    topic,
+                    p.partition,
+                    offset
+                  )
+                end
               end
+            else
+              Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
+                tpl,
+                topic,
+                Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
+              )
             end
-          else
-            Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
-              tpl,
-              topic,
-              Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
-            )
           end
+        rescue Exception
+          # The native list is caller-owned until it is handed to librdkafka. If population raises
+          # partway (e.g. an invalid partition), destroy it here so it is not leaked.
+          Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
+          raise
         end
 
         tpl

data/lib/rdkafka/consumer.rb

@@ -14,14 +14,48 @@ module Rdkafka
     include Enumerable
     include Helpers::Time
     include Helpers::OAuth
+    include Helpers::Metadata
+    include Helpers::ListOffsets
 
     # @private
     # @param native_kafka [NativeKafka] wrapper around the native Kafka consumer handle
     def initialize(native_kafka)
       @native_kafka = native_kafka
+      # Single-element holder shared with the GC finalizer so it can destroy the lazily created
+      # consumer queue without capturing `self` (capturing the consumer in its own finalizer would
+      # pin it and prevent it from ever being collected).
+      @consumer_queue_holder = []
+
+      # Makes sure the consumer is closed (consumer queue destroyed and native client destroyed)
+      # before it gets GCed by Ruby.
+      ObjectSpace.define_finalizer(self, self.class.finalizer(native_kafka, @consumer_queue_holder))
+    end
+
+    # Builds the GC finalizer for a consumer. It mirrors {#close}: close the consumer, destroy the
+    # consumer-queue reference, then destroy the native client. The default `NativeKafka#finalizer`
+    # went straight to `rd_kafka_destroy`, leaving the consumer-queue reference (from
+    # `rd_kafka_queue_get_consumer`, taken by `poll_batch`) dangling - which can make
+    # `rd_kafka_destroy` block inside the finalizer (process hang at GC/shutdown) or leak the
+    # handle.
+    #
+    # @private
+    # @param native_kafka [NativeKafka] the wrapped native client
+    # @param queue_holder [Array] single-element holder carrying the consumer queue pointer
+    #   (or empty)
+    # @return [Proc] finalizer proc that must not reference the consumer instance
+    def self.finalizer(native_kafka, queue_holder)
+      proc do
+        next if native_kafka.closed?
+
+        native_kafka.synchronize do |inner|
+          Rdkafka::Bindings.rd_kafka_consumer_close(inner)
+
+          queue = queue_holder[0]
+          Rdkafka::Bindings.rd_kafka_queue_destroy(queue) if queue
+        end
 
-      # Makes sure, that native kafka gets closed before it gets GCed by Ruby
-      ObjectSpace.define_finalizer(self, native_kafka.finalizer)
+        native_kafka.close
+      end
     end
 
     # Starts the native Kafka polling thread and kicks off the init polling
@@ -93,11 +127,6 @@ module Rdkafka
     # @return [nil]
     # @raise [Rdkafka::ClosedConsumerError] if called on a closed consumer
     #
-    # @note This method holds the inner lock until the queue is empty or `:stop` is returned.
-    #   Other consumer operations will wait until this method returns.
-    # @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
-    # @note Do NOT use this if `consumer_poll_set` was set to `true`
-    #
     # @example Drain all pending events
     #   consumer.events_poll_nb_each { |_count| }
     #
@@ -106,6 +135,10 @@ module Rdkafka
     #   consumer.events_poll_nb_each do |_count|
     #     :stop if monotonic_now >= deadline
     #   end
+    # @note This method holds the inner lock until the queue is empty or `:stop` is returned.
+    #   Other consumer operations will wait until this method returns.
+    # @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
+    # @note Do NOT use this if `consumer_poll_set` was set to `true`
     def events_poll_nb_each
       closed_consumer_check(__method__)
 
@@ -131,14 +164,6 @@ module Rdkafka
     # @raise [Rdkafka::ClosedConsumerError] if called on a closed consumer
     # @raise [Rdkafka::RdkafkaError] if a Kafka error occurs while polling
     #
-    # @note This method uses `rd_kafka_consumer_poll` to fetch messages, unlike
-    #   `events_poll_nb_each` which uses `rd_kafka_poll` for event callbacks (delivery reports,
-    #   statistics, etc.). For consumers, use this method to receive messages and
-    #   `events_poll_nb_each` for processing background events.
-    # @note This method holds the inner lock for the duration. Other consumer operations
-    #   will wait until this method returns.
-    # @note Timeout/max_messages logic should be implemented by the caller
-    #
     # @example Process messages until queue is empty
     #   consumer.poll_nb_each do |message|
     #     process(message)
@@ -151,6 +176,13 @@ module Rdkafka
     #     count += 1
     #     :stop if count >= 10
     #   end
+    # @note This method uses `rd_kafka_consumer_poll` to fetch messages, unlike
+    #   `events_poll_nb_each` which uses `rd_kafka_poll` for event callbacks (delivery reports,
+    #   statistics, etc.). For consumers, use this method to receive messages and
+    #   `events_poll_nb_each` for processing background events.
+    # @note This method holds the inner lock for the duration. Other consumer operations
+    #   will wait until this method returns.
+    # @note Timeout/max_messages logic should be implemented by the caller
     def poll_nb_each
       closed_consumer_check(__method__)
 
@@ -187,6 +219,7 @@ module Rdkafka
         if @consumer_queue
           Rdkafka::Bindings.rd_kafka_queue_destroy(@consumer_queue)
           @consumer_queue = nil
+          @consumer_queue_holder[0] = nil
         end
       end
 
@@ -414,14 +447,18 @@ module Rdkafka
     end
 
     # Return the current positions (offsets) for topics and partitions.
-    # The offset field of each requested partition will be set to the offset of the last consumed message + 1, or nil in case there was no previous message.
+    # The offset field of each requested partition will be set to the offset of the last consumed
+    # message + 1, or nil in case there was no previous message.
     #
-    # @param list [TopicPartitionList, nil] The topic with partitions to get the offsets for or nil to use the current subscription.
+    # @param list [TopicPartitionList, nil] The topic with partitions to get the offsets for or nil
+    #   to use the current subscription.
     #
     # @return [TopicPartitionList]
     #
     # @raise [RdkafkaError] When getting the positions fails.
     def position(list = nil)
+      closed_consumer_check(__method__)
+
       if list.nil?
         list = assignment
       elsif !list.is_a?(TopicPartitionList)
@@ -481,39 +518,76 @@ module Rdkafka
     # possible to create one yourself, in this case you have to provide a list that
     # already contains all the partitions you need the lag for.
     #
+    # The end offsets of all requested partitions are fetched in a single batched
+    # {#list_offsets} query - librdkafka fans it out to the involved partition leaders
+    # internally and concurrently - instead of one blocking {#query_watermark_offsets}
+    # broker roundtrip per partition.
+    #
     # @param topic_partition_list [TopicPartitionList] The list to calculate lag for.
-    # @param watermark_timeout_ms [Integer] The timeout for each query watermark call.
+    # @param watermark_timeout_ms [Integer] The timeout for the batched end-offsets query.
     # @return [Hash{String => Hash{Integer => Integer}}] A hash containing all topics with the lag
     #   per partition
+    # @raise [ClosedConsumerError] when the consumer is closed
     # @raise [RdkafkaError] When querying the broker fails.
     def lag(topic_partition_list, watermark_timeout_ms = Defaults::CONSUMER_LAG_TIMEOUT_MS)
+      closed_consumer_check(__method__)
+
       out = {}
+      request = {}
+      partitions_by_topic = topic_partition_list.to_h
+
+      partitions_by_topic.each do |topic, partitions|
+        out[topic] = {}
 
-      topic_partition_list.to_h.each do |topic, partitions|
-        # Query high watermarks for this topic's partitions
-        # and compare to the offset in the list.
-        topic_out = {}
         partitions.each do |p|
           next if p.offset.nil?
-          _low, high = query_watermark_offsets(
-            topic,
-            p.partition,
-            watermark_timeout_ms
-          )
-          topic_out[p.partition] = high - p.offset
+
+          (request[topic] ||= []) << { partition: p.partition, offset: :latest }
         end
-        out[topic] = topic_out
       end
+
+      return out if request.empty?
+
+      report = begin
+        # The isolation level is forwarded so the end offsets match what the old per-partition
+        # watermark query returned: librdkafka resolves that query with the consumer's configured
+        # `isolation.level` (LSO for the default read_committed), while the admin-style batched
+        # query would otherwise default to read_uncommitted (true high watermark).
+        list_offsets(request, isolation_level: isolation_level)
+          .wait(max_wait_timeout_ms: watermark_timeout_ms)
+      rescue AbstractHandle::WaitTimeoutError
+        # Keep the pre-batching contract: a slow broker surfaced as a timed-out RdkafkaError
+        # from the per-partition watermark query, not as a handle wait timeout.
+        raise RdkafkaError.new(
+          Rdkafka::Bindings::RD_KAFKA_RESP_ERR__TIMED_OUT,
+          "Error querying watermark offsets of '#{request.keys.join(", ")}'"
+        )
+      end
+
+      end_offsets = {}
+      report.offsets.each do |result|
+        (end_offsets[result[:topic]] ||= {})[result[:partition]] = result[:offset]
+      end
+
+      partitions_by_topic.each do |topic, partitions|
+        partitions.each do |p|
+          next if p.offset.nil?
+
+          out[topic][p.partition] = end_offsets.fetch(topic).fetch(p.partition) - p.offset
+        end
+      end
+
       out
     end
 
     # Returns the ClusterId as reported in broker metadata.
     #
+    # @param timeout_ms [Integer] timeout in milliseconds to wait for the cluster id
     # @return [String, nil]
-    def cluster_id
+    def cluster_id(timeout_ms = Defaults::CONSUMER_CLUSTER_ID_TIMEOUT_MS)
       closed_consumer_check(__method__)
       @native_kafka.with_inner do |inner|
-        Rdkafka::Bindings.rd_kafka_clusterid(inner)
+        read_and_free_native_string(inner, Rdkafka::Bindings.rd_kafka_clusterid(inner, timeout_ms))
       end
     end
 
@@ -525,7 +599,7 @@ module Rdkafka
     def member_id
       closed_consumer_check(__method__)
       @native_kafka.with_inner do |inner|
-        Rdkafka::Bindings.rd_kafka_memberid(inner)
+        read_and_free_native_string(inner, Rdkafka::Bindings.rd_kafka_memberid(inner))
       end
     end
 
@@ -534,16 +608,34 @@ module Rdkafka
     # When using this `enable.auto.offset.store` should be set to `false` in the config.
     #
     # @param message [Rdkafka::Consumer::Message] The message which offset will be stored
+    # @param metadata [String, nil] commit metadata string to store alongside the offset
     # @return [nil]
     # @raise [RdkafkaError] When storing the offset fails
-    def store_offset(message)
+    def store_offset(message, metadata = nil)
       closed_consumer_check(__method__)
 
       list = TopicPartitionList.new
-      list.add_topic_and_partitions_with_offsets(
-        message.topic,
-        message.partition => message.offset + 1
-      )
+
+      # For metadata aware commits we build the partition reference directly to save on
+      # objects allocations
+      if metadata
+        list.add_topic_and_partitions_with_offsets(
+          message.topic,
+          [
+            Consumer::Partition.new(
+              message.partition,
+              message.offset + 1,
+              0,
+              metadata
+            )
+          ]
+        )
+      else
+        list.add_topic_and_partitions_with_offsets(
+          message.topic,
+          message.partition => message.offset + 1
+        )
+      end
 
       tpl = list.to_native_tpl
 
@@ -572,8 +664,7 @@ module Rdkafka
     end
 
     # Seek to a particular message by providing the topic, partition and offset.
-    # The next poll on the topic/partition will return the
-    # message at the given offset.
+    # The next poll on the topic/partition will return the message at the given offset.
     #
     # @param topic [String] The topic in which to seek
     # @param partition [Integer] The partition number to seek
@@ -641,8 +732,7 @@ module Rdkafka
 
     # Manually commit the current offsets of this consumer.
     #
-    # To use this set `enable.auto.commit`to `false` to disable automatic triggering
-    # of commits.
+    # To use this set `enable.auto.commit`to `false` to disable automatic triggering of commits.
     #
     # If `enable.auto.offset.store` is set to `true` the offset of the last consumed
     # message for every partition is used. If set to `false` you can use {store_offset} to
@@ -796,7 +886,7 @@ module Rdkafka
     # returns without further waiting.
     #
     # Error events (e.g. `:partition_eof`) are returned inline as {RdkafkaError} objects
-    # rather than raised, so callers receive the complete batch — both messages and errors —
+    # rather than raised, so callers receive the complete batch - both messages and errors -
     # and can decide how to handle each. This is particularly useful when multiple partitions
     # signal EOF simultaneously: all signals appear in the returned array rather than only
     # the first one being raised and the rest silently discarded.
@@ -841,8 +931,17 @@ module Rdkafka
             next
           end
 
-          results << Rdkafka::Consumer::Message.new(native_message)
-          Rdkafka::Bindings.rd_kafka_message_destroy(ptr)
+          begin
+            results << Rdkafka::Consumer::Message.new(native_message)
+          rescue Rdkafka::RdkafkaError => e
+            # A message that fails to build (e.g. a header read error) is surfaced inline as an
+            # error event rather than discarding the whole batch - including the messages already
+            # built - and raising, which silently lost them once their offsets had been stored.
+            results << e
+          ensure
+            Rdkafka::Bindings.rd_kafka_message_destroy(ptr)
+          end
+
           i += 1
         end
       ensure
@@ -862,15 +961,15 @@ module Rdkafka
     # particularly useful in fiber scheduler contexts where GVL release/reacquire
     # overhead is wasteful since we don't expect to wait.
     #
+    # @param timeout_ms [Integer] Timeout waiting for the first message
+    #   (default: 0 for non-blocking)
+    # @param max_items [Integer] Maximum number of messages to return per call
+    # @return [Array<Message, RdkafkaError>] Batch of messages and/or error events in arrival order
+    # @raise [ClosedConsumerError] When called on a closed consumer
     # @note Since the GVL is not released, a non-zero timeout_ms will block all Ruby
     #   threads/fibers for the duration. Use {#poll_batch} if you need a blocking wait.
     #
     # Error events are returned inline as {RdkafkaError} objects; see {#poll_batch} for details.
-    #
-    # @param timeout_ms [Integer] Timeout waiting for the first message (default: 0 for non-blocking)
-    # @param max_items [Integer] Maximum number of messages to return per call
-    # @return [Array<Message, RdkafkaError>] Batch of messages and/or error events in arrival order
-    # @raise [ClosedConsumerError] When called on a closed consumer
     def poll_batch_nb(timeout_ms = 0, max_items: 100)
       closed_consumer_check(__method__)
 
@@ -907,8 +1006,17 @@ module Rdkafka
             next
           end
 
-          results << Rdkafka::Consumer::Message.new(native_message)
-          Rdkafka::Bindings.rd_kafka_message_destroy(ptr)
+          begin
+            results << Rdkafka::Consumer::Message.new(native_message)
+          rescue Rdkafka::RdkafkaError => e
+            # A message that fails to build (e.g. a header read error) is surfaced inline as an
+            # error event rather than discarding the whole batch - including the messages already
+            # built - and raising, which silently lost them once their offsets had been stored.
+            results << e
+          ensure
+            Rdkafka::Bindings.rd_kafka_message_destroy(ptr)
+          end
+
           i += 1
         end
       ensure
@@ -994,12 +1102,63 @@ module Rdkafka
     def closed_consumer_check(method)
       raise Rdkafka::ClosedConsumerError.new(method) if closed?
     end
+    alias_method :closed_check, :closed_consumer_check
+
+    # Reads a librdkafka-allocated string and frees the underlying native buffer.
+    #
+    # librdkafka returns heap-allocated, caller-owned strings from functions like
+    # `rd_kafka_clusterid`/`rd_kafka_memberid`. The buffer must be released with
+    # `rd_kafka_mem_free`, otherwise it leaks on every call.
+    #
+    # @param inner [FFI::Pointer] the native client handle used to allocate the string
+    # @param ptr [FFI::Pointer] the native string pointer (may be null)
+    # @return [String, nil] the copied Ruby string, or nil when the pointer is null
+    def read_and_free_native_string(inner, ptr)
+      return nil if ptr.null?
+
+      ptr.read_string
+    ensure
+      Rdkafka::Bindings.rd_kafka_mem_free(inner, ptr) unless ptr.null?
+    end
+
+    # Reads this consumer's effective `isolation.level` from the live librdkafka configuration
+    # and maps it to the numeric isolation level constant. Memoized: the value cannot change
+    # after client creation.
+    #
+    # @return [Integer] `RD_KAFKA_ISOLATION_LEVEL_READ_COMMITTED` or
+    #   `RD_KAFKA_ISOLATION_LEVEL_READ_UNCOMMITTED`
+    # @raise [Rdkafka::Config::ConfigError] when the property cannot be read
+    def isolation_level
+      @isolation_level ||= @native_kafka.with_inner do |inner|
+        conf = Rdkafka::Bindings.rd_kafka_conf(inner)
+
+        size_ptr = Rdkafka::Bindings::SizePtr.new
+        size_ptr[:value] = 64
+        value_ptr = FFI::MemoryPointer.new(:char, 64)
+
+        result = Rdkafka::Bindings.rd_kafka_conf_get(conf, "isolation.level", value_ptr, size_ptr)
+
+        if result != :config_ok
+          raise Rdkafka::Config::ConfigError.new("Could not read isolation.level: #{result}")
+        end
+
+        if value_ptr.read_string == "read_committed"
+          Rdkafka::Bindings::RD_KAFKA_ISOLATION_LEVEL_READ_COMMITTED
+        else
+          Rdkafka::Bindings::RD_KAFKA_ISOLATION_LEVEL_READ_UNCOMMITTED
+        end
+      end
+    end
 
     # Returns the consumer queue pointer, lazily initialized
     # @return [FFI::Pointer] consumer queue handle
     def consumer_queue
       @consumer_queue ||= @native_kafka.with_inner do |inner|
-        Rdkafka::Bindings.rd_kafka_queue_get_consumer(inner)
+        queue = Rdkafka::Bindings.rd_kafka_queue_get_consumer(inner)
+        # Share the pointer with the finalizer so it is destroyed even if the consumer is GC'd
+        # without an explicit close.
+        @consumer_queue_holder[0] = queue
+        queue
       end
     end
 

data/lib/rdkafka/defaults.rb

@@ -3,19 +3,17 @@
 module Rdkafka
   # Default timeout and timing values used throughout rdkafka-ruby.
   #
-  # All timeout values can be overridden per-call via method parameters.
-  # These constants provide a central place to understand and reference
-  # the default values used across the library.
-  #
-  # @note These are rdkafka-ruby defaults, not librdkafka configuration options.
-  #   For librdkafka options, see:
-  #   https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md
+  # All timeout values can be overridden per-call via method parameters. These constants provide
+  # a central place to understand and reference the default values used across the library.
   #
   # @example Overriding a timeout per-call
   #   consumer.committed(timeout_ms: 5_000)  # Use 5 seconds instead of default 2 seconds
   #
   # @example Checking the default value
   #   Rdkafka::Defaults::CONSUMER_COMMITTED_TIMEOUT_MS  # => 2000
+  # @note These are rdkafka-ruby defaults, not librdkafka configuration options.
+  #   For librdkafka options, see:
+  #   https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md
   module Defaults
     # Consumer timeouts (in milliseconds)
 
@@ -23,6 +21,10 @@ module Rdkafka
     # @see Consumer#committed
     CONSUMER_COMMITTED_TIMEOUT_MS = 2_000
 
+    # Default timeout for fetching the cluster id
+    # @see Consumer#cluster_id
+    CONSUMER_CLUSTER_ID_TIMEOUT_MS = 1_000
+
     # Default timeout for querying watermark offsets
     # @see Consumer#query_watermark_offsets
     CONSUMER_QUERY_WATERMARK_TIMEOUT_MS = 1_000
@@ -91,6 +93,16 @@ module Rdkafka
     # @see Metadata#initialize
     METADATA_RETRY_BACKOFF_BASE_MS = 100
 
+    # Maximum backoff time between metadata retries. Caps the exponential backoff so a long retry
+    # sequence against an unhealthy cluster cannot block the calling thread for minutes.
+    # @see Metadata#initialize
+    METADATA_RETRY_BACKOFF_MAX_MS = 1_000
+
+    # Soft wall-clock budget for the whole metadata retry loop; past it (and past
+    # METADATA_MIN_ATTEMPTS) the loop stops so a synchronous fetch cannot block the caller for long
+    # @see Metadata#initialize
+    METADATA_RETRY_BUDGET_MS = 5_000
+
     # Cache settings (in milliseconds)
 
     # Default time-to-live for cached partition counts
@@ -102,5 +114,10 @@ module Rdkafka
     # Maximum number of metadata fetch retry attempts
     # @see Metadata#initialize
     METADATA_MAX_RETRIES = 10
+
+    # Minimum metadata fetch attempts before the retry budget may end the loop, so a slow broker
+    # (whose requests each consume the full timeout) still gets a few tries
+    # @see Metadata#initialize
+    METADATA_MIN_ATTEMPTS = 3
   end
 end