karafka-rdkafka 0.31.0-aarch64-linux-gnu → 0.32.0-aarch64-linux-gnu

data/lib/rdkafka/config.rb

@@ -156,6 +156,25 @@ module Rdkafka
       )
     end
 
+    # Returns the time-to-live of the process-wide partitions count cache used by
+    # `produce(partition_key:)`.
+    #
+    # @return [Integer] time-to-live in milliseconds
+    def self.partition_count_cache_ttl_ms
+      Rdkafka::Producer.partitions_count_cache.ttl_ms
+    end
+
+    # Sets the time-to-live of the process-wide partitions count cache shared by all the producers.
+    # Cached counts are kept, so it can be changed at any time.
+    #
+    # @param value [Integer] time-to-live in milliseconds
+    #   (defaults to {Defaults::PARTITIONS_COUNT_CACHE_TTL_MS})
+    # @return [void]
+    # @raise [ArgumentError] when the value is not a positive Integer
+    def self.partition_count_cache_ttl_ms=(value)
+      Rdkafka::Producer.partitions_count_cache.ttl_ms = value
+    end
+
     # @private
     def self.opaques
       @@opaques
@@ -180,6 +199,7 @@ module Rdkafka
 
       @config_hash = DEFAULT_CONFIG.merge(config_hash)
       @consumer_rebalance_listener = nil
+      @consumer_offset_commit_listener = nil
       @consumer_poll_set = true
     end
 
@@ -207,6 +227,20 @@ module Rdkafka
     # @return [Object, #on_partitions_assigned, #on_partitions_revoked] listener instance
     attr_writer :consumer_rebalance_listener
 
+    # Get the result of every offset commit, including async commits that the broker rejects
+    #
+    # `#on_offsets_committed` gets an {RdkafkaError} (nil on success) and the
+    # {Consumer::TopicPartitionList} of the commit, whose partitions carry their own errors. It
+    # runs inside {Consumer#poll} (or {Consumer#events_poll} when `consumer_poll_set` is false)
+    # and inside {Consumer#close}, on the thread that calls them.
+    #
+    # `#on_offsets_commit_failed` gets the same arguments, but only for a failed commit: not on
+    # success and not on `no_offset` (nothing new to commit). A listener with only this method
+    # costs nothing for the other commits.
+    #
+    # @return [Object, #on_offsets_committed, #on_offsets_commit_failed] listener instance
+    attr_writer :consumer_offset_commit_listener
+
     # Should we use a single queue for the underlying consumer and events.
     #
     # This is an advanced API that allows for more granular control of the polling process.
@@ -243,6 +277,11 @@ module Rdkafka
         Rdkafka::Bindings.rd_kafka_conf_set_rebalance_cb(config, Rdkafka::Bindings::RebalanceCallback)
       end
 
+      if @consumer_offset_commit_listener
+        opaque.consumer_offset_commit_listener = @consumer_offset_commit_listener
+        Rdkafka::Bindings.rd_kafka_conf_set_offset_commit_cb(config, Rdkafka::Bindings::OffsetCommitCallback)
+      end
+
       build_native_client(
         config,
         :rd_kafka_consumer,
@@ -598,6 +637,7 @@ module Rdkafka
   class Opaque
     attr_accessor :producer
     attr_accessor :consumer_rebalance_listener
+    attr_accessor :consumer_offset_commit_listener
     # Backreference set for share consumers so conf-level callbacks (which receive the opaque)
     # can hand instance-scoped data - currently the librdkafka client name - back to the
     # ShareConsumer, which has no native accessor for it
@@ -635,5 +675,35 @@ module Rdkafka
 
       consumer_rebalance_listener.on_partitions_revoked(list)
     end
+
+    # Invokes the on_offsets_committed callback on the offset commit listener if set
+    # @param error [Rdkafka::RdkafkaError, nil] the commit error or nil on success
+    # @param list [Rdkafka::Consumer::TopicPartitionList] the offsets of the commit
+    def call_on_offsets_committed(error, list)
+      return unless consumer_offset_commit_listener
+      return unless consumer_offset_commit_listener.respond_to?(:on_offsets_committed)
+
+      consumer_offset_commit_listener.on_offsets_committed(error, list)
+    end
+
+    # Invokes the on_offsets_commit_failed callback on the offset commit listener if set
+    # @param error [Rdkafka::RdkafkaError] the commit error
+    # @param list [Rdkafka::Consumer::TopicPartitionList] the offsets of the commit
+    def call_on_offsets_commit_failed(error, list)
+      return unless consumer_offset_commit_listener
+      return unless consumer_offset_commit_listener.respond_to?(:on_offsets_commit_failed)
+
+      consumer_offset_commit_listener.on_offsets_commit_failed(error, list)
+    end
+
+    # @return [Boolean] true if the offset commit listener wants the result of every commit
+    def offsets_committed_listened?
+      consumer_offset_commit_listener.respond_to?(:on_offsets_committed)
+    end
+
+    # @return [Boolean] true if the offset commit listener wants the failed commits
+    def offsets_commit_failed_listened?
+      consumer_offset_commit_listener.respond_to?(:on_offsets_commit_failed)
+    end
   end
 end

data/lib/rdkafka/consumer/allocator.rb

@@ -28,8 +28,6 @@ module Rdkafka
           pointer.put_bytes(0, bytes)
           pointer.put_char(bytes.bytesize, 0) # NUL terminator
 
-          pointer.autorelease = false
-
           pointer
         end
       end

data/lib/rdkafka/consumer/topic_partition_list.rb

@@ -38,7 +38,7 @@ module Rdkafka
       # Calling this method multiple times for the same topic will overwrite the previous
       # configuraton.
       #
-      # @param topic [String] The topic's name
+      # @param topic_name [String] The topic's name
       # @param partitions [Array<Integer>, Range<Integer>, Integer] The topic's partitions or
       #   partition count
       #
@@ -52,14 +52,14 @@ module Rdkafka
       #
       # @example Add a topic with all topics up to a count
       #   tpl.add_topic("topic", 9)
-      def add_topic(topic, partitions = nil)
+      def add_topic(topic_name, partitions = nil)
         if partitions.nil?
-          @data[topic.to_s] = nil
+          @data[topic_name.to_s] = nil
         else
           if partitions.is_a? Integer
             partitions = (0..partitions - 1)
           end
-          @data[topic.to_s] = partitions.map { |p| Partition.new(p, nil, Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR) }
+          @data[topic_name.to_s] = partitions.map { |p| Partition.new(p, nil, Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR) }
         end
       end
 
@@ -67,12 +67,12 @@ module Rdkafka
       # Calling this method multiple times for the same topic will overwrite the previous
       # configuraton.
       #
-      # @param topic [String] The topic's name
+      # @param topic_name [String] The topic's name
       # @param partitions_with_offsets [Hash{Integer => Integer}, Array<Consumer::Partition>] The
       #   topic's partitions and offsets (Hash) or partitions with offsets and metadata (Array)
       # @return [nil]
-      def add_topic_and_partitions_with_offsets(topic, partitions_with_offsets)
-        @data[topic.to_s] = partitions_with_offsets.map do |p, o|
+      def add_topic_and_partitions_with_offsets(topic_name, partitions_with_offsets)
+        @data[topic_name.to_s] = partitions_with_offsets.map do |p, o|
           p.is_a?(Partition) ? p : Partition.new(p, o)
         end
       end

data/lib/rdkafka/consumer.rb

@@ -17,6 +17,9 @@ module Rdkafka
     include Helpers::Metadata
     include Helpers::ListOffsets
 
+    # Reason passed to the error callback when closing the consumer fails
+    CLOSE_ERROR_REASON = "Consumer close failed"
+
     # @private
     # @param native_kafka [NativeKafka] wrapper around the native Kafka consumer handle
     def initialize(native_kafka)
@@ -31,30 +34,61 @@ module Rdkafka
       ObjectSpace.define_finalizer(self, self.class.finalizer(native_kafka, @consumer_queue_holder))
     end
 
-    # Builds the GC finalizer for a consumer. It mirrors {#close}: close the consumer, destroy the
-    # consumer-queue reference, then destroy the native client. The default `NativeKafka#finalizer`
-    # went straight to `rd_kafka_destroy`, leaving the consumer-queue reference (from
-    # `rd_kafka_queue_get_consumer`, taken by `poll_batch`) dangling - which can make
-    # `rd_kafka_destroy` block inside the finalizer (process hang at GC/shutdown) or leak the
-    # handle.
-    #
-    # @private
-    # @param native_kafka [NativeKafka] the wrapped native client
-    # @param queue_holder [Array] single-element holder carrying the consumer queue pointer
-    #   (or empty)
-    # @return [Proc] finalizer proc that must not reference the consumer instance
-    def self.finalizer(native_kafka, queue_holder)
-      proc do
-        next if native_kafka.closed?
-
-        native_kafka.synchronize do |inner|
-          Rdkafka::Bindings.rd_kafka_consumer_close(inner)
-
-          queue = queue_holder[0]
-          Rdkafka::Bindings.rd_kafka_queue_destroy(queue) if queue
+    class << self
+      # Builds the GC finalizer for a consumer. It mirrors {#close}: close the consumer, destroy the
+      # consumer-queue reference, then destroy the native client. The default
+      # `NativeKafka#finalizer` went straight to `rd_kafka_destroy`, leaving the consumer-queue
+      # reference (from `rd_kafka_queue_get_consumer`, taken by `poll_batch`) dangling - which can
+      # make `rd_kafka_destroy` block inside the finalizer (process hang at GC/shutdown) or leak the
+      # handle.
+      #
+      # @private
+      # @param native_kafka [NativeKafka] the wrapped native client
+      # @param queue_holder [Array] single-element holder carrying the consumer queue pointer
+      #   (or empty)
+      # @return [Proc] finalizer proc that must not reference the consumer instance
+      def finalizer(native_kafka, queue_holder)
+        proc do
+          next if native_kafka.closed?
+
+          close_response = nil
+
+          native_kafka.synchronize do |inner|
+            close_response = Rdkafka::Bindings.rd_kafka_consumer_close(inner)
+
+            queue = queue_holder[0]
+            Rdkafka::Bindings.rd_kafka_queue_destroy(queue) if queue
+          end
+
+          report_close_error(native_kafka, close_response)
+
+          native_kafka.close
+        end
+      end
+
+      # Reports a failed `rd_kafka_consumer_close` (e.g. of a fatally-errored consumer) through
+      # {Bindings::ErrorCallback}, the same path as errors emitted by librdkafka, so it reaches
+      # {Config.error_callback} in the shape the callback expects (libraries such as karafka-core
+      # replace `ErrorCallback`). It runs after the close, but before the native client is
+      # destroyed, as the callback reads the client name and fatal error from it. Close runs during
+      # shutdown and from the GC finalizer, so the error is reported, never raised.
+      #
+      # @private
+      # @param native_kafka [NativeKafka] the wrapped native client
+      # @param response [Integer] result code of `rd_kafka_consumer_close`
+      # @return [nil]
+      def report_close_error(native_kafka, response)
+        return if response == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+
+        native_kafka.with_inner do |inner|
+          Rdkafka::Bindings::ErrorCallback.call(inner, response, CLOSE_ERROR_REASON, nil)
         end
 
-        native_kafka.close
+        nil
+      rescue Exception => err
+        Rdkafka::Config.logger.error("Unhandled exception: #{err.class} - #{err.message}")
+
+        nil
       end
     end
 
@@ -114,7 +148,7 @@ module Rdkafka
 
     # Polls for events in a non-blocking loop, yielding the count after each iteration.
     #
-    # This method processes events (stats, errors, etc.) in a single GVL/mutex session,
+    # This method processes events (stats, errors, etc.) in a single GVL session,
     # which is more efficient than repeated individual polls. It uses non-blocking polls
     # internally (no GVL release between polls).
     #
@@ -135,8 +169,9 @@ module Rdkafka
     #   consumer.events_poll_nb_each do |_count|
     #     :stop if monotonic_now >= deadline
     #   end
-    # @note This method holds the inner lock until the queue is empty or `:stop` is returned.
-    #   Other consumer operations will wait until this method returns.
+    # @note This method runs inside `with_inner` and does not hold a lock while it loops.
+    #   Other operations on the same client run concurrently. `close` waits until this
+    #   method returns.
     # @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
     # @note Do NOT use this if `consumer_poll_set` was set to `true`
     def events_poll_nb_each
@@ -153,7 +188,7 @@ module Rdkafka
 
     # Polls for messages in a non-blocking loop, yielding each message to the caller.
     #
-    # This method processes messages in a single GVL/mutex session until the queue is empty
+    # This method processes messages in a single GVL session until the queue is empty
     # or the caller returns `:stop`. It handles the message pointer lifecycle internally,
     # ensuring proper cleanup via `rd_kafka_message_destroy`.
     #
@@ -180,8 +215,9 @@ module Rdkafka
     #   `events_poll_nb_each` which uses `rd_kafka_poll` for event callbacks (delivery reports,
     #   statistics, etc.). For consumers, use this method to receive messages and
     #   `events_poll_nb_each` for processing background events.
-    # @note This method holds the inner lock for the duration. Other consumer operations
-    #   will wait until this method returns.
+    # @note This method runs inside `with_inner` and does not hold a lock while it loops.
+    #   Other operations on the same client run concurrently. `close` waits until this
+    #   method returns.
     # @note Timeout/max_messages logic should be implemented by the caller
     def poll_nb_each
       closed_consumer_check(__method__)
@@ -213,8 +249,10 @@ module Rdkafka
       return if closed?
       ObjectSpace.undefine_finalizer(self)
 
+      close_response = nil
+
       @native_kafka.synchronize do |inner|
-        Rdkafka::Bindings.rd_kafka_consumer_close(inner)
+        close_response = Rdkafka::Bindings.rd_kafka_consumer_close(inner)
 
         if @consumer_queue
           Rdkafka::Bindings.rd_kafka_queue_destroy(@consumer_queue)
@@ -223,6 +261,8 @@ module Rdkafka
         end
       end
 
+      self.class.report_close_error(@native_kafka, close_response)
+
       @native_kafka.close
     end
 
@@ -469,12 +509,12 @@ module Rdkafka
 
     # Query broker for low (oldest/beginning) and high (newest/end) offsets for a partition.
     #
-    # @param topic [String] The topic to query
+    # @param topic_name [String] The topic to query
     # @param partition [Integer] The partition to query
     # @param timeout_ms [Integer] The timeout for querying the broker
     # @return [Integer] The low and high watermark
     # @raise [RdkafkaError] When querying the broker fails.
-    def query_watermark_offsets(topic, partition, timeout_ms = Defaults::CONSUMER_QUERY_WATERMARK_TIMEOUT_MS)
+    def query_watermark_offsets(topic_name, partition, timeout_ms = Defaults::CONSUMER_QUERY_WATERMARK_TIMEOUT_MS)
       closed_consumer_check(__method__)
 
       low = FFI::MemoryPointer.new(:int64, 1)
@@ -483,13 +523,13 @@ module Rdkafka
       @native_kafka.with_inner do |inner|
         response = Rdkafka::Bindings.rd_kafka_query_watermark_offsets(
           inner,
-          topic,
+          topic_name,
           partition,
           low,
           high,
           timeout_ms
         )
-        Rdkafka::RdkafkaError.validate!(response, "Error querying watermark offsets for partition #{partition} of #{topic}", client_ptr: inner)
+        Rdkafka::RdkafkaError.validate!(response, "Error querying watermark offsets for partition #{partition} of #{topic_name}", client_ptr: inner)
       end
 
       [low.read_array_of_int64(1).first, high.read_array_of_int64(1).first]
@@ -651,12 +691,12 @@ module Rdkafka
     # Seek to a particular message by providing the topic, partition and offset. The next poll on
     # the topic/partition will return the message at the given offset.
     #
-    # @param topic [String] The topic in which to seek
+    # @param topic_name [String] The topic in which to seek
     # @param partition [Integer] The partition number to seek
     # @param offset [Integer] The partition offset to seek
     # @return [nil]
     # @raise [RdkafkaError] When seeking fails
-    def seek_by(topic, partition, offset)
+    def seek_by(topic_name, partition, offset)
       closed_consumer_check(__method__)
 
       # rd_kafka_offset_store is one of the few calls that does not support
@@ -664,7 +704,7 @@ module Rdkafka
       native_topic = @native_kafka.with_inner do |inner|
         Rdkafka::Bindings.rd_kafka_topic_new(
           inner,
-          topic,
+          topic_name,
           nil
         )
       end

data/lib/rdkafka/defaults.rb

@@ -114,12 +114,14 @@ module Rdkafka
     # @see Config#admin
     NATIVE_KAFKA_POLL_TIMEOUT_MS = 100
 
-    # Sleep interval used in NativeKafka#synchronize wait loop
-    # @see NativeKafka#synchronize
-    NATIVE_KAFKA_SYNCHRONIZE_SLEEP_INTERVAL_MS = 10
-
     # TTL for partitions count cache entries (30 seconds)
     # @see Producer::PartitionsCountCache
     PARTITIONS_COUNT_CACHE_TTL_MS = 30_000
+
+    # How long an expired partitions count is served after a refresh failed on a transient
+    # (e.g. transport) error, before the next refresh attempt. Longer than one failed metadata
+    # refresh (up to ~7 s with retries), so produces are not blocked most of the time
+    # @see Producer::PartitionsCountCache#get
+    PARTITIONS_COUNT_CACHE_STALE_BACKOFF_MS = 10_000
   end
 end

data/lib/rdkafka/native_kafka.rb

@@ -25,21 +25,15 @@ module Rdkafka
       @access_mutex = Mutex.new
       # Lock around internal polling
       @poll_mutex = Mutex.new
-      # Lock around decrementing the operations in progress counter
-      # We have two mutexes - one for increment (`@access_mutex`) and one for decrement mutex
-      # because they serve different purposes:
-      #
-      #   - `@access_mutex` allows us to lock the execution and make sure that any operation within
-      #      the `#synchronize` is the only one running and that there are no other running
-      #      operations.
-      #   - `@decrement_mutex` ensures, that our decrement operation is thread-safe for any Ruby
-      #     implementation.
-      #
-      # We do not use the same mutex, because it could create a deadlock when an already
-      # incremented operation cannot decrement because `@access_lock` is now owned by a different
-      # thread in a synchronized mode and the synchronized mode is waiting on the decrement.
-      @decrement_mutex = Mutex.new
-      # counter for operations in progress using inner
+      # Signalled when `#synchronize` ends and, only while `#synchronize` waits, when the operations
+      # in progress counter reaches zero: no one else waits for zero, so other decrements skip the
+      # broadcast. `#synchronize` waits on it with `@access_mutex` released, so operations that
+      # started before it can still decrement the counter.
+      @operations_done = ConditionVariable.new
+      # Set while `#synchronize` runs, so no new operation starts until it ends
+      @synchronizing = false
+      # counter for operations in progress using inner, changed only under `@access_mutex`, so no
+      # update is lost on Rubies without a GVL (JRuby)
       @operations_in_progress = 0
 
       @run_polling_thread = run_polling_thread
@@ -63,8 +57,7 @@ module Rdkafka
         Rdkafka::Bindings.rd_kafka_poll(@inner, 0)
 
         if @run_polling_thread
-          # Start thread to poll client for delivery callbacks,
-          # not used in consumer.
+          # Start thread to poll client for delivery callbacks, not used in consumer.
           @polling_thread = Thread.new do
             loop do
               @poll_mutex.synchronize do
@@ -90,15 +83,29 @@ module Rdkafka
     # @return [Object] the result of the block
     # @raise [ClosedInnerError] when the inner handle is nil
     def with_inner
-      if @access_mutex.owned?
+      owned = @access_mutex.owned?
+
+      if owned
         @operations_in_progress += 1
       else
-        @access_mutex.synchronize { @operations_in_progress += 1 }
+        @access_mutex.synchronize do
+          @operations_done.wait(@access_mutex) while @synchronizing
+          @operations_in_progress += 1
+        end
       end
 
-      @inner.nil? ? raise(ClosedInnerError) : yield(@inner)
-    ensure
-      @decrement_mutex.synchronize { @operations_in_progress -= 1 }
+      begin
+        @inner.nil? ? raise(ClosedInnerError) : yield(@inner)
+      ensure
+        if owned
+          @operations_in_progress -= 1
+        else
+          @access_mutex.synchronize do
+            @operations_in_progress -= 1
+            @operations_done.broadcast if @synchronizing && @operations_in_progress.zero?
+          end
+        end
+      end
     end
 
     # Executes a block while holding exclusive access to the native Kafka handle
@@ -107,13 +114,21 @@ module Rdkafka
     # @return [Object] the result of the block
     def synchronize(&block)
       @access_mutex.synchronize do
-        # Wait for any commands using the inner to finish
-        # This can take a while on blocking operations like polling but is essential not to proceed
-        # with certain types of operations like resources destruction as it can cause the process
-        # to hang or crash
-        sleep(Defaults::NATIVE_KAFKA_SYNCHRONIZE_SLEEP_INTERVAL_MS / 1_000.0) until @operations_in_progress.zero?
-
-        with_inner(&block)
+        @operations_done.wait(@access_mutex) while @synchronizing
+        @synchronizing = true
+
+        begin
+          # Wait for any commands using the inner to finish
+          # This can take a while on blocking operations like polling but is essential not to
+          # proceed with certain types of operations like resources destruction as it can cause the
+          # process to hang or crash
+          @operations_done.wait(@access_mutex) until @operations_in_progress.zero?
+
+          with_inner(&block)
+        ensure
+          @synchronizing = false
+          @operations_done.broadcast
+        end
       end
     end
 
@@ -210,8 +225,7 @@ module Rdkafka
           # Indicate to polling thread that we're closing
           @polling_thread[:closing] = true
 
-          # Wait for the polling thread to finish up,
-          # this can be aborted in practice if this
+          # Wait for the polling thread to finish up, this can be aborted in practice if this
           # code runs from a finalizer.
           @polling_thread.join
         end