karafka-rdkafka 0.30.2-aarch64-linux-gnu → 0.32.0-aarch64-linux-gnu

data/lib/rdkafka/consumer.rb

@@ -17,6 +17,9 @@ module Rdkafka
     include Helpers::Metadata
     include Helpers::ListOffsets
 
+    # Reason passed to the error callback when closing the consumer fails
+    CLOSE_ERROR_REASON = "Consumer close failed"
+
     # @private
     # @param native_kafka [NativeKafka] wrapper around the native Kafka consumer handle
     def initialize(native_kafka)
@@ -31,30 +34,61 @@ module Rdkafka
       ObjectSpace.define_finalizer(self, self.class.finalizer(native_kafka, @consumer_queue_holder))
     end
 
-    # Builds the GC finalizer for a consumer. It mirrors {#close}: close the consumer, destroy the
-    # consumer-queue reference, then destroy the native client. The default `NativeKafka#finalizer`
-    # went straight to `rd_kafka_destroy`, leaving the consumer-queue reference (from
-    # `rd_kafka_queue_get_consumer`, taken by `poll_batch`) dangling - which can make
-    # `rd_kafka_destroy` block inside the finalizer (process hang at GC/shutdown) or leak the
-    # handle.
-    #
-    # @private
-    # @param native_kafka [NativeKafka] the wrapped native client
-    # @param queue_holder [Array] single-element holder carrying the consumer queue pointer
-    #   (or empty)
-    # @return [Proc] finalizer proc that must not reference the consumer instance
-    def self.finalizer(native_kafka, queue_holder)
-      proc do
-        next if native_kafka.closed?
-
-        native_kafka.synchronize do |inner|
-          Rdkafka::Bindings.rd_kafka_consumer_close(inner)
-
-          queue = queue_holder[0]
-          Rdkafka::Bindings.rd_kafka_queue_destroy(queue) if queue
+    class << self
+      # Builds the GC finalizer for a consumer. It mirrors {#close}: close the consumer, destroy the
+      # consumer-queue reference, then destroy the native client. The default
+      # `NativeKafka#finalizer` went straight to `rd_kafka_destroy`, leaving the consumer-queue
+      # reference (from `rd_kafka_queue_get_consumer`, taken by `poll_batch`) dangling - which can
+      # make `rd_kafka_destroy` block inside the finalizer (process hang at GC/shutdown) or leak the
+      # handle.
+      #
+      # @private
+      # @param native_kafka [NativeKafka] the wrapped native client
+      # @param queue_holder [Array] single-element holder carrying the consumer queue pointer
+      #   (or empty)
+      # @return [Proc] finalizer proc that must not reference the consumer instance
+      def finalizer(native_kafka, queue_holder)
+        proc do
+          next if native_kafka.closed?
+
+          close_response = nil
+
+          native_kafka.synchronize do |inner|
+            close_response = Rdkafka::Bindings.rd_kafka_consumer_close(inner)
+
+            queue = queue_holder[0]
+            Rdkafka::Bindings.rd_kafka_queue_destroy(queue) if queue
+          end
+
+          report_close_error(native_kafka, close_response)
+
+          native_kafka.close
+        end
+      end
+
+      # Reports a failed `rd_kafka_consumer_close` (e.g. of a fatally-errored consumer) through
+      # {Bindings::ErrorCallback}, the same path as errors emitted by librdkafka, so it reaches
+      # {Config.error_callback} in the shape the callback expects (libraries such as karafka-core
+      # replace `ErrorCallback`). It runs after the close, but before the native client is
+      # destroyed, as the callback reads the client name and fatal error from it. Close runs during
+      # shutdown and from the GC finalizer, so the error is reported, never raised.
+      #
+      # @private
+      # @param native_kafka [NativeKafka] the wrapped native client
+      # @param response [Integer] result code of `rd_kafka_consumer_close`
+      # @return [nil]
+      def report_close_error(native_kafka, response)
+        return if response == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+
+        native_kafka.with_inner do |inner|
+          Rdkafka::Bindings::ErrorCallback.call(inner, response, CLOSE_ERROR_REASON, nil)
         end
 
-        native_kafka.close
+        nil
+      rescue Exception => err
+        Rdkafka::Config.logger.error("Unhandled exception: #{err.class} - #{err.message}")
+
+        nil
       end
     end
 
@@ -114,7 +148,7 @@ module Rdkafka
 
     # Polls for events in a non-blocking loop, yielding the count after each iteration.
     #
-    # This method processes events (stats, errors, etc.) in a single GVL/mutex session,
+    # This method processes events (stats, errors, etc.) in a single GVL session,
     # which is more efficient than repeated individual polls. It uses non-blocking polls
     # internally (no GVL release between polls).
     #
@@ -135,8 +169,9 @@ module Rdkafka
     #   consumer.events_poll_nb_each do |_count|
     #     :stop if monotonic_now >= deadline
     #   end
-    # @note This method holds the inner lock until the queue is empty or `:stop` is returned.
-    #   Other consumer operations will wait until this method returns.
+    # @note This method runs inside `with_inner` and does not hold a lock while it loops.
+    #   Other operations on the same client run concurrently. `close` waits until this
+    #   method returns.
     # @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
     # @note Do NOT use this if `consumer_poll_set` was set to `true`
     def events_poll_nb_each
@@ -153,7 +188,7 @@ module Rdkafka
 
     # Polls for messages in a non-blocking loop, yielding each message to the caller.
     #
-    # This method processes messages in a single GVL/mutex session until the queue is empty
+    # This method processes messages in a single GVL session until the queue is empty
     # or the caller returns `:stop`. It handles the message pointer lifecycle internally,
     # ensuring proper cleanup via `rd_kafka_message_destroy`.
     #
@@ -180,8 +215,9 @@ module Rdkafka
     #   `events_poll_nb_each` which uses `rd_kafka_poll` for event callbacks (delivery reports,
     #   statistics, etc.). For consumers, use this method to receive messages and
     #   `events_poll_nb_each` for processing background events.
-    # @note This method holds the inner lock for the duration. Other consumer operations
-    #   will wait until this method returns.
+    # @note This method runs inside `with_inner` and does not hold a lock while it loops.
+    #   Other operations on the same client run concurrently. `close` waits until this
+    #   method returns.
     # @note Timeout/max_messages logic should be implemented by the caller
     def poll_nb_each
       closed_consumer_check(__method__)
@@ -213,8 +249,10 @@ module Rdkafka
       return if closed?
       ObjectSpace.undefine_finalizer(self)
 
+      close_response = nil
+
       @native_kafka.synchronize do |inner|
-        Rdkafka::Bindings.rd_kafka_consumer_close(inner)
+        close_response = Rdkafka::Bindings.rd_kafka_consumer_close(inner)
 
         if @consumer_queue
           Rdkafka::Bindings.rd_kafka_queue_destroy(@consumer_queue)
@@ -223,6 +261,8 @@ module Rdkafka
         end
       end
 
+      self.class.report_close_error(@native_kafka, close_response)
+
       @native_kafka.close
     end
 
@@ -469,12 +509,12 @@ module Rdkafka
 
     # Query broker for low (oldest/beginning) and high (newest/end) offsets for a partition.
     #
-    # @param topic [String] The topic to query
+    # @param topic_name [String] The topic to query
     # @param partition [Integer] The partition to query
     # @param timeout_ms [Integer] The timeout for querying the broker
     # @return [Integer] The low and high watermark
     # @raise [RdkafkaError] When querying the broker fails.
-    def query_watermark_offsets(topic, partition, timeout_ms = Defaults::CONSUMER_QUERY_WATERMARK_TIMEOUT_MS)
+    def query_watermark_offsets(topic_name, partition, timeout_ms = Defaults::CONSUMER_QUERY_WATERMARK_TIMEOUT_MS)
       closed_consumer_check(__method__)
 
       low = FFI::MemoryPointer.new(:int64, 1)
@@ -483,13 +523,13 @@ module Rdkafka
       @native_kafka.with_inner do |inner|
         response = Rdkafka::Bindings.rd_kafka_query_watermark_offsets(
           inner,
-          topic,
+          topic_name,
           partition,
           low,
           high,
           timeout_ms
         )
-        Rdkafka::RdkafkaError.validate!(response, "Error querying watermark offsets for partition #{partition} of #{topic}", client_ptr: inner)
+        Rdkafka::RdkafkaError.validate!(response, "Error querying watermark offsets for partition #{partition} of #{topic_name}", client_ptr: inner)
       end
 
       [low.read_array_of_int64(1).first, high.read_array_of_int64(1).first]
@@ -651,12 +691,12 @@ module Rdkafka
     # Seek to a particular message by providing the topic, partition and offset. The next poll on
     # the topic/partition will return the message at the given offset.
     #
-    # @param topic [String] The topic in which to seek
+    # @param topic_name [String] The topic in which to seek
     # @param partition [Integer] The partition number to seek
     # @param offset [Integer] The partition offset to seek
     # @return [nil]
     # @raise [RdkafkaError] When seeking fails
-    def seek_by(topic, partition, offset)
+    def seek_by(topic_name, partition, offset)
       closed_consumer_check(__method__)
 
       # rd_kafka_offset_store is one of the few calls that does not support
@@ -664,7 +704,7 @@ module Rdkafka
       native_topic = @native_kafka.with_inner do |inner|
         Rdkafka::Bindings.rd_kafka_topic_new(
           inner,
-          topic,
+          topic_name,
           nil
         )
       end

data/lib/rdkafka/defaults.rb

@@ -114,12 +114,14 @@ module Rdkafka
     # @see Config#admin
     NATIVE_KAFKA_POLL_TIMEOUT_MS = 100
 
-    # Sleep interval used in NativeKafka#synchronize wait loop
-    # @see NativeKafka#synchronize
-    NATIVE_KAFKA_SYNCHRONIZE_SLEEP_INTERVAL_MS = 10
-
     # TTL for partitions count cache entries (30 seconds)
     # @see Producer::PartitionsCountCache
     PARTITIONS_COUNT_CACHE_TTL_MS = 30_000
+
+    # How long an expired partitions count is served after a refresh failed on a transient
+    # (e.g. transport) error, before the next refresh attempt. Longer than one failed metadata
+    # refresh (up to ~7 s with retries), so produces are not blocked most of the time
+    # @see Producer::PartitionsCountCache#get
+    PARTITIONS_COUNT_CACHE_STALE_BACKOFF_MS = 10_000
   end
 end

data/lib/rdkafka/native_kafka.rb

@@ -25,21 +25,15 @@ module Rdkafka
       @access_mutex = Mutex.new
       # Lock around internal polling
       @poll_mutex = Mutex.new
-      # Lock around decrementing the operations in progress counter
-      # We have two mutexes - one for increment (`@access_mutex`) and one for decrement mutex
-      # because they serve different purposes:
-      #
-      #   - `@access_mutex` allows us to lock the execution and make sure that any operation within
-      #      the `#synchronize` is the only one running and that there are no other running
-      #      operations.
-      #   - `@decrement_mutex` ensures, that our decrement operation is thread-safe for any Ruby
-      #     implementation.
-      #
-      # We do not use the same mutex, because it could create a deadlock when an already
-      # incremented operation cannot decrement because `@access_lock` is now owned by a different
-      # thread in a synchronized mode and the synchronized mode is waiting on the decrement.
-      @decrement_mutex = Mutex.new
-      # counter for operations in progress using inner
+      # Signalled when `#synchronize` ends and, only while `#synchronize` waits, when the operations
+      # in progress counter reaches zero: no one else waits for zero, so other decrements skip the
+      # broadcast. `#synchronize` waits on it with `@access_mutex` released, so operations that
+      # started before it can still decrement the counter.
+      @operations_done = ConditionVariable.new
+      # Set while `#synchronize` runs, so no new operation starts until it ends
+      @synchronizing = false
+      # counter for operations in progress using inner, changed only under `@access_mutex`, so no
+      # update is lost on Rubies without a GVL (JRuby)
       @operations_in_progress = 0
 
       @run_polling_thread = run_polling_thread
@@ -63,8 +57,7 @@ module Rdkafka
         Rdkafka::Bindings.rd_kafka_poll(@inner, 0)
 
         if @run_polling_thread
-          # Start thread to poll client for delivery callbacks,
-          # not used in consumer.
+          # Start thread to poll client for delivery callbacks, not used in consumer.
           @polling_thread = Thread.new do
             loop do
               @poll_mutex.synchronize do
@@ -90,15 +83,29 @@ module Rdkafka
     # @return [Object] the result of the block
     # @raise [ClosedInnerError] when the inner handle is nil
     def with_inner
-      if @access_mutex.owned?
+      owned = @access_mutex.owned?
+
+      if owned
         @operations_in_progress += 1
       else
-        @access_mutex.synchronize { @operations_in_progress += 1 }
+        @access_mutex.synchronize do
+          @operations_done.wait(@access_mutex) while @synchronizing
+          @operations_in_progress += 1
+        end
       end
 
-      @inner.nil? ? raise(ClosedInnerError) : yield(@inner)
-    ensure
-      @decrement_mutex.synchronize { @operations_in_progress -= 1 }
+      begin
+        @inner.nil? ? raise(ClosedInnerError) : yield(@inner)
+      ensure
+        if owned
+          @operations_in_progress -= 1
+        else
+          @access_mutex.synchronize do
+            @operations_in_progress -= 1
+            @operations_done.broadcast if @synchronizing && @operations_in_progress.zero?
+          end
+        end
+      end
     end
 
     # Executes a block while holding exclusive access to the native Kafka handle
@@ -107,13 +114,21 @@ module Rdkafka
     # @return [Object] the result of the block
     def synchronize(&block)
       @access_mutex.synchronize do
-        # Wait for any commands using the inner to finish
-        # This can take a while on blocking operations like polling but is essential not to proceed
-        # with certain types of operations like resources destruction as it can cause the process
-        # to hang or crash
-        sleep(Defaults::NATIVE_KAFKA_SYNCHRONIZE_SLEEP_INTERVAL_MS / 1_000.0) until @operations_in_progress.zero?
-
-        with_inner(&block)
+        @operations_done.wait(@access_mutex) while @synchronizing
+        @synchronizing = true
+
+        begin
+          # Wait for any commands using the inner to finish
+          # This can take a while on blocking operations like polling but is essential not to
+          # proceed with certain types of operations like resources destruction as it can cause the
+          # process to hang or crash
+          @operations_done.wait(@access_mutex) until @operations_in_progress.zero?
+
+          with_inner(&block)
+        ensure
+          @synchronizing = false
+          @operations_done.broadcast
+        end
       end
     end
 
@@ -210,8 +225,7 @@ module Rdkafka
           # Indicate to polling thread that we're closing
           @polling_thread[:closing] = true
 
-          # Wait for the polling thread to finish up,
-          # this can be aborted in practice if this
+          # Wait for the polling thread to finish up, this can be aborted in practice if this
           # code runs from a finalizer.
           @polling_thread.join
         end

data/lib/rdkafka/producer/delivery_handle.rb

@@ -7,7 +7,10 @@ module Rdkafka
       layout :pending, :bool,
         :response, :int,
         :partition, :int,
-        :offset, :int64
+        :offset, :int64,
+        :status, :int,
+        :latency, :int64,
+        :broker_id, :int32
 
       # @return [Object, nil] label set during message production or nil by default
       attr_accessor :label
@@ -24,23 +27,22 @@ module Rdkafka
 
       # @return [DeliveryReport] a report on the delivery of the message
       def create_result
-        if self[:response] == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
-          DeliveryReport.new(
-            self[:partition],
-            self[:offset],
-            topic,
-            nil,
-            label
-          )
+        error = if self[:response] == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+          nil
         else
-          DeliveryReport.new(
-            self[:partition],
-            self[:offset],
-            topic,
-            Rdkafka::RdkafkaError.build(self[:response]),
-            label
-          )
+          Rdkafka::RdkafkaError.build(self[:response])
         end
+
+        DeliveryReport.new(
+          self[:partition],
+          self[:offset],
+          topic,
+          error,
+          label,
+          status: self[:status],
+          latency: self[:latency],
+          broker_id: self[:broker_id]
+        )
       end
     end
   end

data/lib/rdkafka/producer/delivery_report.rb

@@ -25,6 +25,26 @@ module Rdkafka
       # @return [Object, nil] label set during message production or nil by default
       attr_reader :label
 
+      # Persistence status of the message, one of `Bindings::RD_KAFKA_MSG_STATUS_*`:
+      #   - `RD_KAFKA_MSG_STATUS_NOT_PERSISTED` - the message was not written to the broker. A retry
+      #     risks reordering, but not a duplicate.
+      #   - `RD_KAFKA_MSG_STATUS_POSSIBLY_PERSISTED` - the message may have been written (e.g. the
+      #     request timed out before an ack). A retry risks reordering and a duplicate.
+      #   - `RD_KAFKA_MSG_STATUS_PERSISTED` - the broker acknowledged the message. Trust this only
+      #     with `acks=all`.
+      # `nil` when the report was not built from a delivery callback.
+      # @return [Integer, nil]
+      attr_reader :status
+
+      # Time from the `produce` call until the delivery report, in microseconds, or `nil` when
+      #   not available
+      # @return [Integer, nil]
+      attr_reader :latency
+
+      # Id of the broker the message was produced to, or `nil` when not known
+      # @return [Integer, nil]
+      attr_reader :broker_id
+
       # We alias the `#topic_name` under `#topic` to make this consistent with `Consumer::Message`
       # where the topic name is under `#topic` method. That way we have a consistent name that
       # is present in both places
@@ -38,12 +58,42 @@ module Rdkafka
       # @param topic_name [String, nil] topic name
       # @param error [Integer, nil] error code if any
       # @param label [Object, nil] user-defined label
-      def initialize(partition, offset, topic_name = nil, error = nil, label = nil)
+      # @param status [Integer, nil] `Bindings::RD_KAFKA_MSG_STATUS_*` persistence status
+      # @param latency [Integer, nil] produce latency in microseconds, negative when not available
+      # @param broker_id [Integer, nil] broker id, negative when not known
+      def initialize(
+        partition,
+        offset,
+        topic_name = nil,
+        error = nil,
+        label = nil,
+        status: nil,
+        latency: nil,
+        broker_id: nil
+      )
         @partition = partition
         @offset = offset
         @topic_name = topic_name
         @error = error
         @label = label
+        @status = status
+        @latency = (latency.nil? || latency.negative?) ? nil : latency
+        @broker_id = (broker_id.nil? || broker_id.negative?) ? nil : broker_id
+      end
+
+      # @return [Boolean] true when the broker acknowledged the message
+      def persisted?
+        status == Bindings::RD_KAFKA_MSG_STATUS_PERSISTED
+      end
+
+      # @return [Boolean] true when the message may have been written, so a retry may duplicate it
+      def possibly_persisted?
+        status == Bindings::RD_KAFKA_MSG_STATUS_POSSIBLY_PERSISTED
+      end
+
+      # @return [Boolean] true when the message was not written, so a retry does not duplicate it
+      def not_persisted?
+        status == Bindings::RD_KAFKA_MSG_STATUS_NOT_PERSISTED
       end
     end
   end