rdkafka 0.22.2 → 0.29.0

data/lib/rdkafka/native_kafka.rb

@@ -4,7 +4,13 @@ module Rdkafka
   # @private
   # A wrapper around a native kafka that polls and cleanly exits
   class NativeKafka
-    def initialize(inner, run_polling_thread:, opaque:, auto_start: true, timeout_ms: 100)
+    # Creates a new NativeKafka wrapper
+    # @param inner [FFI::Pointer] pointer to the native Kafka handle
+    # @param run_polling_thread [Boolean] whether to run a background polling thread
+    # @param opaque [Rdkafka::Opaque] opaque object for callback context
+    # @param auto_start [Boolean] whether to start the polling thread automatically
+    # @param timeout_ms [Integer] poll timeout in milliseconds
+    def initialize(inner, run_polling_thread:, opaque:, auto_start: true, timeout_ms: Defaults::NATIVE_KAFKA_POLL_TIMEOUT_MS)
       @inner = inner
       @opaque = opaque
       # Lock around external access
@@ -37,6 +43,8 @@ module Rdkafka
       @closing = false
     end
 
+    # Starts the polling thread if configured
+    # @return [nil]
     def start
       synchronize do
         return if @started
@@ -47,8 +55,7 @@ module Rdkafka
         Rdkafka::Bindings.rd_kafka_poll(@inner, 0)
 
         if @run_polling_thread
-          # Start thread to poll client for delivery callbacks,
-          # not used in consumer.
+          # Start thread to poll client for delivery callbacks, not used in consumer.
           @polling_thread = Thread.new do
             loop do
               @poll_mutex.synchronize do
@@ -62,13 +69,17 @@ module Rdkafka
             end
           end
 
-          @polling_thread.name = "rdkafka.native_kafka##{Rdkafka::Bindings.rd_kafka_name(@inner).gsub('rdkafka', '')}"
+          @polling_thread.name = "rdkafka.native_kafka##{Rdkafka::Bindings.rd_kafka_name(@inner).gsub("rdkafka", "")}"
           @polling_thread.abort_on_exception = true
           @polling_thread[:closing] = false
         end
       end
     end
 
+    # Executes a block with the inner native Kafka handle
+    # @yield [FFI::Pointer] the inner native Kafka handle
+    # @return [Object] the result of the block
+    # @raise [ClosedInnerError] when the inner handle is nil
     def with_inner
       if @access_mutex.owned?
         @operations_in_progress += 1
@@ -81,27 +92,100 @@ module Rdkafka
       @decrement_mutex.synchronize { @operations_in_progress -= 1 }
     end
 
+    # Executes a block while holding exclusive access to the native Kafka handle
+    # @param block [Proc] block to execute with the native handle
+    # @yield [FFI::Pointer] the inner native Kafka handle
+    # @return [Object] the result of the block
     def synchronize(&block)
       @access_mutex.synchronize do
         # Wait for any commands using the inner to finish
         # This can take a while on blocking operations like polling but is essential not to proceed
         # with certain types of operations like resources destruction as it can cause the process
         # to hang or crash
-        sleep(0.01) until @operations_in_progress.zero?
+        sleep(Defaults::NATIVE_KAFKA_SYNCHRONIZE_SLEEP_INTERVAL_MS / 1000.0) until @operations_in_progress.zero?
 
         with_inner(&block)
       end
     end
 
+    # Returns a finalizer proc for closing this native Kafka handle
+    # @return [Proc] finalizer proc
     def finalizer
       ->(_) { close }
     end
 
+    # Returns whether this native Kafka handle is closed or closing
+    # @return [Boolean] true if closed or closing
     def closed?
       @closing || @inner.nil?
     end
 
-    def close(object_id=nil)
+    # Enable IO event notifications on the main queue
+    # Librdkafka will write to your FD when the queue transitions from empty to non-empty
+    #
+    # @note This method is incompatible with background polling threads.
+    #   If background polling is enabled, use manual polling instead (e.g., consumer.poll)
+    #
+    # @param fd [Integer] your file descriptor (from IO.pipe or eventfd)
+    # @param payload [String] data to write to fd when queue has data (default: "\x01")
+    # @return [nil]
+    # @raise [ClosedInnerError] when the handle is closed
+    # @raise [RuntimeError] when background polling thread is active
+    #
+    # @example
+    #   # Create your own signaling FD
+    #   signal_r, signal_w = IO.pipe
+    #   native_kafka.enable_main_queue_io_events(signal_w.fileno)
+    #
+    #   # Monitor it with select
+    #   readable, = IO.select([signal_r], nil, nil, timeout)
+    #   if readable
+    #     consumer.poll(0)  # Get messages
+    #   end
+    def enable_main_queue_io_events(fd, payload = "\x01")
+      if @run_polling_thread
+        raise "Cannot enable IO events while background polling thread is active. " \
+          "Either disable background polling by setting run_polling_thread: false, " \
+          "or use manual polling with consumer.poll() instead of the FD API."
+      end
+
+      with_inner do |inner|
+        queue_ptr = Bindings.rd_kafka_queue_get_main(inner)
+        Bindings.rd_kafka_queue_io_event_enable(queue_ptr, fd, payload, payload.bytesize)
+        Bindings.rd_kafka_queue_destroy(queue_ptr)
+      end
+    end
+
+    # Enable IO event notifications on the background queue
+    # Librdkafka will write to your FD when the background queue transitions from empty to non-empty
+    #
+    # @note This method is incompatible with background polling threads.
+    #   If background polling is enabled, use manual polling instead (e.g., consumer.poll)
+    #
+    # @param fd [Integer] your file descriptor (from IO.pipe or eventfd)
+    # @param payload [String] data to write to fd when queue has data (default: "\x01")
+    # @return [nil]
+    # @raise [ClosedInnerError] when the handle is closed
+    # @raise [RuntimeError] when background polling thread is active
+    def enable_background_queue_io_events(fd, payload = "\x01")
+      if @run_polling_thread
+        raise "Cannot enable IO events while background polling thread is active. " \
+          "Either disable background polling by setting run_polling_thread: false, " \
+          "or use manual polling with consumer.poll() instead of the FD API."
+      end
+
+      with_inner do |inner|
+        queue_ptr = Bindings.rd_kafka_queue_get_background(inner)
+        Bindings.rd_kafka_queue_io_event_enable(queue_ptr, fd, payload, payload.bytesize)
+        Bindings.rd_kafka_queue_destroy(queue_ptr)
+      end
+    end
+
+    # Closes the native Kafka handle and cleans up resources
+    # @param object_id [Integer, nil] optional object ID (unused, for finalizer compatibility)
+    # @yield optional block to execute before destroying the handle
+    # @return [nil]
+    def close(object_id = nil)
       return if closed?
 
       synchronize do
@@ -112,8 +196,7 @@ module Rdkafka
           # Indicate to polling thread that we're closing
           @polling_thread[:closing] = true
 
-          # Wait for the polling thread to finish up,
-          # this can be aborted in practice if this
+          # Wait for the polling thread to finish up, this can be aborted in practice if this
           # code runs from a finalizer.
           @polling_thread.join
         end

data/lib/rdkafka/producer/delivery_handle.rb

@@ -6,16 +6,16 @@ module Rdkafka
     # producing a message.
     class DeliveryHandle < Rdkafka::AbstractHandle
       layout :pending, :bool,
-             :response, :int,
-             :partition, :int,
-             :offset, :int64,
-             :topic_name, :pointer
+        :response, :int,
+        :partition, :int,
+        :offset, :int64
 
       # @return [Object, nil] label set during message production or nil by default
       attr_accessor :label
 
       # @return [String] topic where we are trying to send the message
-      # We use this instead of reading from `topic_name` pointer to save on memory allocations
+      # Set in `#produce`, where the topic is known upfront. Keeping it as a Ruby attribute
+      # spares a per-message native string copy in the delivery callback.
       attr_accessor :topic
 
       # @return [String] the name of the operation (e.g. "delivery")
@@ -31,7 +31,7 @@ module Rdkafka
           # For part of errors, we will not get a topic name reference and in cases like this
           # we should not return it
           topic,
-          self[:response] != 0 ? RdkafkaError.new(self[:response]) : nil,
+          (self[:response] != 0) ? RdkafkaError.new(self[:response]) : nil,
           label
         )
       end

data/lib/rdkafka/producer/delivery_report.rb

@@ -12,8 +12,8 @@ module Rdkafka
       # @return [Integer]
       attr_reader :offset
 
-      # The name of the topic this message was produced to or nil in case of reports with errors
-      # where topic was not reached.
+      # The name of the topic this message was produced to or nil in case delivery failed and we
+      #   we not able to get the topic reference
       #
       # @return [String, nil]
       attr_reader :topic_name
@@ -30,10 +30,14 @@ module Rdkafka
       # is present in both places
       #
       # We do not remove the original `#topic_name` because of backwards compatibility
-      alias topic topic_name
-
-      private
-
+      alias_method :topic, :topic_name
+
+      # @private
+      # @param partition [Integer] partition number
+      # @param offset [Integer] message offset
+      # @param topic_name [String, nil] topic name
+      # @param error [Integer, nil] error code if any
+      # @param label [Object, nil] user-defined label
       def initialize(partition, offset, topic_name = nil, error = nil, label = nil)
         @partition = partition
         @offset = offset

data/lib/rdkafka/producer/partitions_count_cache.rb

@@ -17,9 +17,9 @@ module Rdkafka
     #
     # 2. Edge case handling
     #    If a user configures `statistics.interval.ms` much higher than the default cache TTL
-    #    (30 seconds), the cache will still function correctly. When statistics updates don't
-    #    occur frequently enough, the cache entries will expire naturally, triggering a
-    #    blocking refresh when needed.
+    #    ({Defaults::PARTITIONS_COUNT_CACHE_TTL_MS}ms), the cache will still function correctly.
+    #    When statistics updates don't occur frequently enough, the cache entries will expire
+    #    naturally, triggering a blocking refresh when needed.
     #
     # 3. User configuration awareness
     #    The cache respects user-defined settings. If `topic.metadata.refresh.interval.ms` is
@@ -37,31 +37,40 @@ module Rdkafka
     #    contention in multi-threaded environments while ensuring data consistency.
     #
     # 6. Topic recreation handling
-    #    If a topic is deleted and recreated with fewer partitions, the cache will continue to
-    #    report the higher count until either the TTL expires or the process is restarted. This
-    #    design choice simplifies the implementation while relying on librdkafka's error handling
-    #    for edge cases. In production environments, topic recreation with different partition
-    #    counts is typically accompanied by application restarts to handle structural changes.
-    #    This also aligns with the previous cache implementation.
+    #    If a topic is deleted and recreated with fewer partitions, the cache keeps reporting the
+    #    higher count only until the entry's TTL expires. The first refresh after expiry performs
+    #    an authoritative metadata read and adopts the lower count. Within the TTL window a lower
+    #    value is still ignored, so a transient or racy lower read cannot clobber a correct higher
+    #    count.
     class PartitionsCountCache
       include Helpers::Time
 
-      # Default time-to-live for cached partition counts in seconds
-      #
-      # @note This default was chosen to balance freshness of metadata with performance
-      #   optimization. Most Kafka cluster topology changes are planned operations, making 30
-      #   seconds a reasonable compromise.
-      DEFAULT_TTL = 30
-
       # Creates a new partition count cache
       #
-      # @param ttl [Integer] Time-to-live in seconds for cached values
-      def initialize(ttl = DEFAULT_TTL)
+      # @param ttl [Integer, nil] DEPRECATED: Use ttl_ms instead.
+      #   Time-to-live in seconds for cached values. Will be removed in v1.0.0.
+      # @param ttl_ms [Integer, nil] Time-to-live in milliseconds for cached values.
+      #   Defaults to {Defaults::PARTITIONS_COUNT_CACHE_TTL_MS}.
+      def initialize(ttl = :not_provided, ttl_ms: :not_provided)
         @counts = {}
         @mutex_hash = {}
         # Used only for @mutex_hash access to ensure thread-safety when creating new mutexes
         @mutex_for_hash = Mutex.new
-        @ttl = ttl
+
+        # Determine which TTL value to use
+        if ttl != :not_provided && ttl_ms != :not_provided
+          warn "DEPRECATION WARNING: Both ttl and ttl_ms were provided to PartitionsCountCache. " \
+               "Using ttl_ms. The ttl parameter is deprecated and will be removed in v1.0.0."
+          @ttl_ms = ttl_ms
+        elsif ttl != :not_provided
+          warn "DEPRECATION WARNING: ttl (seconds) parameter for PartitionsCountCache is deprecated. " \
+               "Use ttl_ms (milliseconds) instead. This parameter will be removed in v1.0.0."
+          @ttl_ms = (ttl * 1000).to_i
+        elsif ttl_ms == :not_provided
+          @ttl_ms = Defaults::PARTITIONS_COUNT_CACHE_TTL_MS
+        else
+          @ttl_ms = ttl_ms
+        end
       end
 
       # Reads partition count for a topic with automatic refresh when expired
@@ -81,28 +90,14 @@ module Rdkafka
         current_info = @counts[topic]
 
         if current_info.nil? || expired?(current_info[0])
+          # The cached entry is missing or expired, so the block performs an authoritative metadata
+          # read. We hand it to `set`, which adopts a higher count always and a lower count once the
+          # entry has expired (e.g. the topic was recreated with fewer partitions). We then return
+          # whatever `set` settled on so a concurrent refresh that wrote a higher value still wins.
           new_count = yield
+          set(topic, new_count)
 
-          if current_info.nil?
-            # No existing data, create a new entry with mutex
-            set(topic, new_count)
-
-            return new_count
-          else
-            current_count = current_info[1]
-
-            if new_count > current_count
-              # Higher value needs mutex to update both timestamp and count
-              set(topic, new_count)
-
-              return new_count
-            else
-              # Same or lower value, just update timestamp without mutex
-              refresh_timestamp(topic)
-
-              return current_count
-            end
-          end
+          return @counts[topic][1]
         end
 
         current_info[1]
@@ -123,8 +118,11 @@ module Rdkafka
         # First check outside mutex to avoid unnecessary locking
         current_info = @counts[topic]
 
-        # For lower values, we don't update count but might need to refresh timestamp
-        if current_info && new_count < current_info[1]
+        # Within the TTL window a lower value is treated as a stale/racy read and ignored, since
+        # partition counts only grow during normal operation. Once the entry has expired a lower
+        # value is an authoritative refresh (e.g. the topic was recreated with fewer partitions),
+        # so we fall through and adopt it below.
+        if current_info && new_count < current_info[1] && !expired?(current_info[0])
           refresh_timestamp(topic)
 
           return
@@ -137,18 +135,16 @@ module Rdkafka
 
           if current_info.nil?
             # Create new entry
-            @counts[topic] = [monotonic_now, new_count]
+            @counts[topic] = [monotonic_now_ms, new_count]
+          elsif new_count > current_info[1] || expired?(current_info[0])
+            # A higher count always wins; a lower count is accepted only when the existing entry
+            # has expired, so a concurrent fresh higher value (which reset the timestamp) is never
+            # clobbered by a stale lower one.
+            current_info[0] = monotonic_now_ms
+            current_info[1] = new_count
           else
-            current_count = current_info[1]
-
-            if new_count > current_count
-              # Update to higher count value
-              current_info[0] = monotonic_now
-              current_info[1] = new_count
-            else
-              # Same or lower count, update timestamp only
-              current_info[0] = monotonic_now
-            end
+            # Same or lower count within the TTL window: refresh the timestamp only
+            current_info[0] = monotonic_now_ms
           end
         end
       end
@@ -201,15 +197,15 @@ module Rdkafka
         return unless current_info
 
         # Update the timestamp in-place
-        current_info[0] = monotonic_now
+        current_info[0] = monotonic_now_ms
       end
 
       # Check if a timestamp has expired based on the TTL
       #
-      # @param timestamp [Float] Monotonic timestamp to check
+      # @param timestamp [Integer] Monotonic timestamp in milliseconds to check
       # @return [Boolean] true if expired, false otherwise
       def expired?(timestamp)
-        monotonic_now - timestamp > @ttl
+        monotonic_now_ms - timestamp > @ttl_ms
       end
     end
   end