karafka-rdkafka 0.27.1-aarch64-linux-gnu → 0.28.0-aarch64-linux-gnu

data/lib/rdkafka/consumer/topic_partition_list.rb

@@ -136,51 +136,61 @@ module Rdkafka
 
       # Create a native tpl with the contents of this object added.
       #
-      # The pointer will be cleaned by `rd_kafka_topic_partition_list_destroy` when GC releases it.
+      # The returned pointer is caller-owned and must be released with
+      # `rd_kafka_topic_partition_list_destroy`. It is not cleaned up by GC.
       #
       # @private
       # @return [FFI::Pointer]
       def to_native_tpl
         tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(count)
 
-        @data.each do |topic, partitions|
-          if partitions
-            partitions.each do |p|
-              ref = Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
-                tpl,
-                topic,
-                p.partition
-              )
-
-              # Remove the respond to check after karafka 2.3.0 is released
-              if p.respond_to?(:metadata) && p.metadata
-                part = Rdkafka::Bindings::TopicPartition.new(ref)
-                str_ptr = FFI::MemoryPointer.from_string(p.metadata)
-                # released here:
-                # https://github.com/confluentinc/librdkafka/blob/e03d3bb91ed92a38f38d9806b8d8deffe78a1de5/src/rdkafka_partition.c#L2682C18-L2682C18
-                str_ptr.autorelease = false
-                part[:metadata] = str_ptr
-                part[:metadata_size] = p.metadata.bytesize
-              end
-
-              if p.offset
-                offset = p.offset.is_a?(Time) ? p.offset.to_f * 1_000 : p.offset
-
-                Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(
+        begin
+          @data.each do |topic, partitions|
+            if partitions
+              partitions.each do |p|
+                ref = Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
                   tpl,
                   topic,
-                  p.partition,
-                  offset
+                  p.partition
                 )
+
+                # Remove the respond to check after karafka 2.3.0 is released
+                if p.respond_to?(:metadata) && p.metadata
+                  part = Rdkafka::Bindings::TopicPartition.new(ref)
+                  str_ptr = FFI::MemoryPointer.from_string(p.metadata)
+                  # released here:
+                  # https://github.com/confluentinc/librdkafka/blob/e03d3bb91ed92a38f38d9806b8d8deffe78a1de5/src/rdkafka_partition.c#L2682C18-L2682C18
+                  str_ptr.autorelease = false
+                  part[:metadata] = str_ptr
+                  part[:metadata_size] = p.metadata.bytesize
+                end
+
+                if p.offset
+                  offset = p.offset.is_a?(Time) ? p.offset.to_f * 1_000 : p.offset
+
+                  Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(
+                    tpl,
+                    topic,
+                    p.partition,
+                    offset
+                  )
+                end
               end
+            else
+              Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
+                tpl,
+                topic,
+                Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
+              )
             end
-          else
-            Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
-              tpl,
-              topic,
-              Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
-            )
           end
+        rescue Exception
+          # The native list is caller-owned and only handed back on success. If population raises
+          # partway (e.g. non-string metadata, or an offset FFI can't coerce to int64) we must
+          # destroy it here, otherwise the half-built list leaks.
+          Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
+
+          raise
         end
 
         tpl

data/lib/rdkafka/consumer.rb

@@ -14,14 +14,46 @@ module Rdkafka
     include Enumerable
     include Helpers::Time
     include Helpers::OAuth
+    include Helpers::Metadata
+    include Helpers::ListOffsets
 
     # @private
     # @param native_kafka [NativeKafka] wrapper around the native Kafka consumer handle
     def initialize(native_kafka)
       @native_kafka = native_kafka
+      # Single-element holder shared with the GC finalizer so it can destroy the lazily created
+      # consumer queue without capturing `self` (capturing the consumer in its own finalizer would
+      # pin it and prevent it from ever being collected).
+      @consumer_queue_holder = []
+
+      # Makes sure the consumer is closed (consumer queue destroyed and native client destroyed)
+      # before it gets GCed by Ruby.
+      ObjectSpace.define_finalizer(self, self.class.finalizer(native_kafka, @consumer_queue_holder))
+    end
+
+    # Builds the GC finalizer for a consumer. It mirrors {#close}: close the consumer, destroy the
+    # consumer-queue reference, then destroy the native client. The default `NativeKafka#finalizer`
+    # went straight to `rd_kafka_destroy`, leaving the consumer-queue reference (from
+    # `rd_kafka_queue_get_consumer`, taken by `poll_batch`) dangling - which can make
+    # `rd_kafka_destroy` block inside the finalizer (process hang at GC/shutdown) or leak the handle.
+    #
+    # @private
+    # @param native_kafka [NativeKafka] the wrapped native client
+    # @param queue_holder [Array] single-element holder carrying the consumer queue pointer (or empty)
+    # @return [Proc] finalizer proc that must not reference the consumer instance
+    def self.finalizer(native_kafka, queue_holder)
+      proc do
+        next if native_kafka.closed?
+
+        native_kafka.synchronize do |inner|
+          Rdkafka::Bindings.rd_kafka_consumer_close(inner)
+
+          queue = queue_holder[0]
+          Rdkafka::Bindings.rd_kafka_queue_destroy(queue) if queue
+        end
 
-      # Makes sure, that native kafka gets closed before it gets GCed by Ruby
-      ObjectSpace.define_finalizer(self, native_kafka.finalizer)
+        native_kafka.close
+      end
     end
 
     # Starts the native Kafka polling thread and kicks off the init polling
@@ -187,6 +219,7 @@ module Rdkafka
         if @consumer_queue
           Rdkafka::Bindings.rd_kafka_queue_destroy(@consumer_queue)
           @consumer_queue = nil
+          @consumer_queue_holder[0] = nil
         end
       end
 
@@ -412,6 +445,8 @@ module Rdkafka
     #
     # @raise [RdkafkaError] When getting the positions fails.
     def position(list = nil)
+      closed_consumer_check(__method__)
+
       if list.nil?
         list = assignment
       elsif !list.is_a?(TopicPartitionList)
@@ -466,39 +501,77 @@ module Rdkafka
     # possible to create one yourself, in this case you have to provide a list that
     # already contains all the partitions you need the lag for.
     #
+    # The end offsets of all requested partitions are fetched in a single batched
+    # {#list_offsets} query - librdkafka fans it out to the involved partition leaders
+    # internally and concurrently - instead of one blocking {#query_watermark_offsets}
+    # broker roundtrip per partition.
+    #
     # @param topic_partition_list [TopicPartitionList] The list to calculate lag for.
-    # @param watermark_timeout_ms [Integer] The timeout for each query watermark call.
+    # @param watermark_timeout_ms [Integer] The timeout for the batched end-offsets query.
     # @return [Hash{String => Hash{Integer => Integer}}] A hash containing all topics with the lag
     #   per partition
+    # @raise [ClosedConsumerError] when the consumer is closed
     # @raise [RdkafkaError] When querying the broker fails.
     def lag(topic_partition_list, watermark_timeout_ms = Defaults::CONSUMER_LAG_TIMEOUT_MS)
+      closed_consumer_check(__method__)
+
       out = {}
+      request = {}
+      partitions_by_topic = topic_partition_list.to_h
+
+      partitions_by_topic.each do |topic, partitions|
+        out[topic] = {}
+
+        partitions.each do |p|
+          next if p.offset.nil?
+
+          (request[topic] ||= []) << { partition: p.partition, offset: :latest }
+        end
+      end
+
+      return out if request.empty?
+
+      report = begin
+        # The isolation level is forwarded so the end offsets match what the old per-partition
+        # watermark query returned: librdkafka resolves that query with the consumer's configured
+        # `isolation.level` (LSO for the default read_committed), while the admin-style batched
+        # query would otherwise default to read_uncommitted (true high watermark).
+        list_offsets(request, isolation_level: isolation_level)
+          .wait(max_wait_timeout_ms: watermark_timeout_ms)
+      rescue AbstractHandle::WaitTimeoutError
+        # Keep the pre-batching contract: a slow broker surfaced as a timed-out RdkafkaError
+        # from the per-partition watermark query, not as a handle wait timeout.
+        raise RdkafkaError.new(
+          Rdkafka::Bindings::RD_KAFKA_RESP_ERR__TIMED_OUT,
+          "Error querying watermark offsets of '#{request.keys.join(", ")}'"
+        )
+      end
 
-      topic_partition_list.to_h.each do |topic, partitions|
-        # Query high watermarks for this topic's partitions
-        # and compare to the offset in the list.
-        topic_out = {}
+      end_offsets = {}
+      report.offsets.each do |result|
+        (end_offsets[result[:topic]] ||= {})[result[:partition]] = result[:offset]
+      end
+
+      partitions_by_topic.each do |topic, partitions|
         partitions.each do |p|
           next if p.offset.nil?
-          _low, high = query_watermark_offsets(
-            topic,
-            p.partition,
-            watermark_timeout_ms
-          )
-          topic_out[p.partition] = high - p.offset
+
+          out[topic][p.partition] = end_offsets.fetch(topic).fetch(p.partition) - p.offset
         end
-        out[topic] = topic_out
       end
+
       out
     end
 
     # Returns the ClusterId as reported in broker metadata.
     #
-    # @return [String, nil]
-    def cluster_id
+    # @param timeout_ms [Integer] how long to block waiting for metadata when the cluster id is
+    #   not already cached (0 for a non-blocking call)
+    # @return [String, nil] the cluster id, or nil if it could not be retrieved in time
+    def cluster_id(timeout_ms = Defaults::CONSUMER_CLUSTER_ID_TIMEOUT_MS)
       closed_consumer_check(__method__)
       @native_kafka.with_inner do |inner|
-        Rdkafka::Bindings.rd_kafka_clusterid(inner)
+        read_and_free_native_string(inner, Rdkafka::Bindings.rd_kafka_clusterid(inner, timeout_ms))
       end
     end
 
@@ -510,7 +583,7 @@ module Rdkafka
     def member_id
       closed_consumer_check(__method__)
       @native_kafka.with_inner do |inner|
-        Rdkafka::Bindings.rd_kafka_memberid(inner)
+        read_and_free_native_string(inner, Rdkafka::Bindings.rd_kafka_memberid(inner))
       end
     end
 
@@ -801,16 +874,21 @@ module Rdkafka
     # is available, librdkafka fills the buffer with whatever is immediately ready and
     # returns without further waiting.
     #
+    # Error events (e.g. `:partition_eof`) are returned inline as {RdkafkaError} objects
+    # rather than raised, so callers receive the complete batch — both messages and errors —
+    # and can decide how to handle each. This is particularly useful when multiple partitions
+    # signal EOF simultaneously: all signals appear in the returned array rather than only
+    # the first one being raised and the rest silently discarded.
+    #
     # @param timeout_ms [Integer] Timeout waiting for the first message (-1 for infinite)
     # @param max_items [Integer] Maximum number of messages to return per call
-    # @return [Array<Message>] Array of messages (empty if none available within timeout)
-    # @raise [RdkafkaError] When a consumed message contains an error
+    # @return [Array<Message, RdkafkaError>] Batch of messages and/or error events in arrival order
     # @raise [ClosedConsumerError] When called on a closed consumer
     def poll_batch(timeout_ms, max_items: 100)
       closed_consumer_check(__method__)
 
       buffer = batch_buffer(max_items)
-      messages = []
+      results = []
 
       count = @native_kafka.with_inner do |_inner|
         Rdkafka::Bindings.rd_kafka_consume_batch_queue(
@@ -821,7 +899,7 @@ module Rdkafka
         )
       end
 
-      return messages if count <= 0
+      return results if count <= 0
 
       i = 0
       begin
@@ -836,13 +914,23 @@ module Rdkafka
           native_message = Rdkafka::Bindings::Message.new(ptr)
 
           if native_message[:err] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
-            @native_kafka.with_inner do |inner|
-              Rdkafka::RdkafkaError.validate!(native_message, client_ptr: inner)
-            end
+            results << build_batch_error(native_message)
+            Rdkafka::Bindings.rd_kafka_message_destroy(ptr)
+            i += 1
+            next
+          end
+
+          begin
+            results << Rdkafka::Consumer::Message.new(native_message)
+          rescue Rdkafka::RdkafkaError => e
+            # A message that fails to build (e.g. a header read error) is surfaced inline as an
+            # error event rather than discarding the whole batch - including the messages already
+            # built - and raising, which silently lost them once their offsets had been stored.
+            results << e
+          ensure
+            Rdkafka::Bindings.rd_kafka_message_destroy(ptr)
           end
 
-          messages << Rdkafka::Consumer::Message.new(native_message)
-          Rdkafka::Bindings.rd_kafka_message_destroy(ptr)
           i += 1
         end
       ensure
@@ -853,7 +941,7 @@ module Rdkafka
         end
       end
 
-      messages
+      results
     end
 
     # Poll for a batch of messages without releasing the GVL (Global VM Lock).
@@ -865,16 +953,17 @@ module Rdkafka
     # @note Since the GVL is not released, a non-zero timeout_ms will block all Ruby
     #   threads/fibers for the duration. Use {#poll_batch} if you need a blocking wait.
     #
+    # Error events are returned inline as {RdkafkaError} objects; see {#poll_batch} for details.
+    #
     # @param timeout_ms [Integer] Timeout waiting for the first message (default: 0 for non-blocking)
     # @param max_items [Integer] Maximum number of messages to return per call
-    # @return [Array<Message>] Array of messages (empty if none available within timeout)
-    # @raise [RdkafkaError] When a consumed message contains an error
+    # @return [Array<Message, RdkafkaError>] Batch of messages and/or error events in arrival order
     # @raise [ClosedConsumerError] When called on a closed consumer
     def poll_batch_nb(timeout_ms = 0, max_items: 100)
       closed_consumer_check(__method__)
 
       buffer = batch_buffer(max_items)
-      messages = []
+      results = []
 
       count = @native_kafka.with_inner do |_inner|
         Rdkafka::Bindings.rd_kafka_consume_batch_queue_nb(
@@ -885,7 +974,7 @@ module Rdkafka
         )
       end
 
-      return messages if count <= 0
+      return results if count <= 0
 
       i = 0
       begin
@@ -900,13 +989,23 @@ module Rdkafka
           native_message = Rdkafka::Bindings::Message.new(ptr)
 
           if native_message[:err] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
-            @native_kafka.with_inner do |inner|
-              Rdkafka::RdkafkaError.validate!(native_message, client_ptr: inner)
-            end
+            results << build_batch_error(native_message)
+            Rdkafka::Bindings.rd_kafka_message_destroy(ptr)
+            i += 1
+            next
+          end
+
+          begin
+            results << Rdkafka::Consumer::Message.new(native_message)
+          rescue Rdkafka::RdkafkaError => e
+            # A message that fails to build (e.g. a header read error) is surfaced inline as an
+            # error event rather than discarding the whole batch - including the messages already
+            # built - and raising, which silently lost them once their offsets had been stored.
+            results << e
+          ensure
+            Rdkafka::Bindings.rd_kafka_message_destroy(ptr)
           end
 
-          messages << Rdkafka::Consumer::Message.new(native_message)
-          Rdkafka::Bindings.rd_kafka_message_destroy(ptr)
           i += 1
         end
       ensure
@@ -917,7 +1016,7 @@ module Rdkafka
         end
       end
 
-      messages
+      results
     end
 
     # Poll for new messages and yield for each received one. Iteration
@@ -986,18 +1085,83 @@ module Rdkafka
 
     private
 
+    # Copies a librdkafka-allocated C string into a Ruby string and frees the native buffer.
+    #
+    # `rd_kafka_memberid`/`rd_kafka_clusterid` return a string the caller owns and must release
+    # with `rd_kafka_mem_free`; without this the buffer leaks on every call.
+    #
+    # @param inner [FFI::Pointer] the native client handle (needed to free the buffer)
+    # @param ptr [FFI::Pointer] the librdkafka-allocated string, possibly NULL
+    # @return [String, nil] the copied string, or nil when the pointer was NULL
+    def read_and_free_native_string(inner, ptr)
+      return nil if ptr.null?
+
+      ptr.read_string
+    ensure
+      Rdkafka::Bindings.rd_kafka_mem_free(inner, ptr) unless ptr.null?
+    end
+
     # Checks if the consumer is closed and raises an error if so
     # @param method [Symbol] name of the calling method for error context
     # @raise [ClosedConsumerError] when the consumer is closed
     def closed_consumer_check(method)
       raise Rdkafka::ClosedConsumerError.new(method) if closed?
     end
+    alias_method :closed_check, :closed_consumer_check
+
+    # Reads this consumer's effective `isolation.level` from the live librdkafka configuration
+    # and maps it to the numeric isolation level constant. Memoized: the value cannot change
+    # after client creation.
+    #
+    # @return [Integer] `RD_KAFKA_ISOLATION_LEVEL_READ_COMMITTED` or
+    #   `RD_KAFKA_ISOLATION_LEVEL_READ_UNCOMMITTED`
+    # @raise [Rdkafka::Config::ConfigError] when the property cannot be read
+    def isolation_level
+      @isolation_level ||= @native_kafka.with_inner do |inner|
+        conf = Rdkafka::Bindings.rd_kafka_conf(inner)
+
+        size_ptr = Rdkafka::Bindings::SizePtr.new
+        size_ptr[:value] = 64
+        value_ptr = FFI::MemoryPointer.new(:char, 64)
+
+        result = Rdkafka::Bindings.rd_kafka_conf_get(conf, "isolation.level", value_ptr, size_ptr)
+
+        if result != :config_ok
+          raise Rdkafka::Config::ConfigError.new("Could not read isolation.level: #{result}")
+        end
+
+        if value_ptr.read_string == "read_committed"
+          Rdkafka::Bindings::RD_KAFKA_ISOLATION_LEVEL_READ_COMMITTED
+        else
+          Rdkafka::Bindings::RD_KAFKA_ISOLATION_LEVEL_READ_UNCOMMITTED
+        end
+      end
+    end
 
     # Returns the consumer queue pointer, lazily initialized
     # @return [FFI::Pointer] consumer queue handle
     def consumer_queue
       @consumer_queue ||= @native_kafka.with_inner do |inner|
-        Rdkafka::Bindings.rd_kafka_queue_get_consumer(inner)
+        queue = Rdkafka::Bindings.rd_kafka_queue_get_consumer(inner)
+        # Share the pointer with the finalizer so it is destroyed even if the consumer is GC'd
+        # without an explicit close.
+        @consumer_queue_holder[0] = queue
+        queue
+      end
+    end
+
+    # Builds an RdkafkaError from a native message, promoting to a fatal error when
+    # librdkafka flagged a fatal condition on the underlying client handle.
+    # @param native_message [Rdkafka::Bindings::Message] the errored native message
+    # @return [RdkafkaError]
+    def build_batch_error(native_message)
+      err = Rdkafka::RdkafkaError.build(native_message)
+      if err && err.rdkafka_response == Rdkafka::Bindings::RD_KAFKA_RESP_ERR__FATAL
+        @native_kafka.with_inner do |inner|
+          Rdkafka::RdkafkaError.build_fatal(inner, fallback_error_code: err.rdkafka_response)
+        end
+      else
+        err
       end
     end
 

data/lib/rdkafka/defaults.rb

@@ -22,6 +22,10 @@ module Rdkafka
     # @see Consumer#offsets_for_times
     CONSUMER_OFFSETS_FOR_TIMES_TIMEOUT_MS = 1_000
 
+    # Consumer timeout for fetching the cluster id when it is not already cached from metadata
+    # @see Consumer#cluster_id
+    CONSUMER_CLUSTER_ID_TIMEOUT_MS = 1_000
+
     # Consumer timeout for poll operations (used in each iteration)
     # @see Consumer#each
     CONSUMER_POLL_TIMEOUT_MS = 250
@@ -55,14 +59,30 @@ module Rdkafka
     # @see Admin#metadata
     METADATA_TIMEOUT_MS = 2_000
 
-    # Maximum retries for metadata requests on transient errors
+    # Hard ceiling on metadata fetch attempts on transient errors (backstop; the retry budget
+    # normally ends the loop first)
     # @see Metadata#initialize
     METADATA_MAX_RETRIES = 10
 
+    # Minimum metadata fetch attempts before the retry budget may end the loop, so a slow broker
+    # (whose requests each consume the full timeout) still gets a few tries
+    # @see Metadata#initialize
+    METADATA_MIN_ATTEMPTS = 3
+
+    # Soft wall-clock budget for the whole metadata retry loop; past it (and past
+    # METADATA_MIN_ATTEMPTS) the loop stops so a synchronous fetch cannot block the caller for long
+    # @see Metadata#initialize
+    METADATA_RETRY_BUDGET_MS = 5_000
+
     # Base backoff time for metadata retry (100ms = 0.1s)
     # @see Metadata#initialize
     METADATA_RETRY_BACKOFF_BASE_MS = 100
 
+    # Maximum backoff time between metadata retries. Caps the exponential backoff so a long retry
+    # sequence against an unhealthy cluster cannot block the calling thread for minutes.
+    # @see Metadata#initialize
+    METADATA_RETRY_BACKOFF_MAX_MS = 1_000
+
     # Default wait timeout for operation handles
     # @see AbstractHandle#wait
     HANDLE_WAIT_TIMEOUT_MS = 60_000

data/lib/rdkafka/error.rb

@@ -35,8 +35,9 @@ module Rdkafka
       # @param response_ptr [FFI::Pointer] Pointer to rd_kafka_error_t
       # @param message_prefix [String, nil] Optional prefix for the error message
       # @param broker_message [String, nil] Optional broker error message
+      # @param instance_name [String, nil] Optional name of the rdkafka instance
       # @return [RdkafkaError, false] Error instance or false if no error
-      def build_from_c(response_ptr, message_prefix = nil, broker_message: nil)
+      def build_from_c(response_ptr, message_prefix = nil, broker_message: nil, instance_name: nil)
         code = Rdkafka::Bindings.rd_kafka_error_code(response_ptr)
 
         return false if code == Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
@@ -54,7 +55,8 @@ module Rdkafka
           broker_message: message,
           fatal: fatal,
           retryable: retryable,
-          abortable: abortable
+          abortable: abortable,
+          instance_name: instance_name
         )
       end
 
@@ -92,10 +94,16 @@ module Rdkafka
             response_ptr_or_code[:err],
             message_prefix,
             broker_message: broker_message,
-            details: details
+            details: details,
+            instance_name: instance_name
           )
         else
-          build_from_c(response_ptr_or_code, message_prefix)
+          build_from_c(
+            response_ptr_or_code,
+            message_prefix,
+            broker_message: broker_message,
+            instance_name: instance_name
+          )
         end
       end