rdkafka 0.22.2 → 0.27.0

data/lib/rdkafka/producer/partitions_count_cache.rb

@@ -17,9 +17,9 @@ module Rdkafka
     #
     # 2. Edge case handling
     #    If a user configures `statistics.interval.ms` much higher than the default cache TTL
-    #    (30 seconds), the cache will still function correctly. When statistics updates don't
-    #    occur frequently enough, the cache entries will expire naturally, triggering a
-    #    blocking refresh when needed.
+    #    ({Defaults::PARTITIONS_COUNT_CACHE_TTL_MS}ms), the cache will still function correctly.
+    #    When statistics updates don't occur frequently enough, the cache entries will expire
+    #    naturally, triggering a blocking refresh when needed.
     #
     # 3. User configuration awareness
     #    The cache respects user-defined settings. If `topic.metadata.refresh.interval.ms` is
@@ -46,22 +46,32 @@ module Rdkafka
     class PartitionsCountCache
       include Helpers::Time
 
-      # Default time-to-live for cached partition counts in seconds
-      #
-      # @note This default was chosen to balance freshness of metadata with performance
-      #   optimization. Most Kafka cluster topology changes are planned operations, making 30
-      #   seconds a reasonable compromise.
-      DEFAULT_TTL = 30
-
       # Creates a new partition count cache
       #
-      # @param ttl [Integer] Time-to-live in seconds for cached values
-      def initialize(ttl = DEFAULT_TTL)
+      # @param ttl [Integer, nil] DEPRECATED: Use ttl_ms instead.
+      #   Time-to-live in seconds for cached values. Will be removed in v1.0.0.
+      # @param ttl_ms [Integer, nil] Time-to-live in milliseconds for cached values.
+      #   Defaults to {Defaults::PARTITIONS_COUNT_CACHE_TTL_MS}.
+      def initialize(ttl = :not_provided, ttl_ms: :not_provided)
         @counts = {}
         @mutex_hash = {}
         # Used only for @mutex_hash access to ensure thread-safety when creating new mutexes
         @mutex_for_hash = Mutex.new
-        @ttl = ttl
+
+        # Determine which TTL value to use
+        if ttl != :not_provided && ttl_ms != :not_provided
+          warn "DEPRECATION WARNING: Both ttl and ttl_ms were provided to PartitionsCountCache. " \
+               "Using ttl_ms. The ttl parameter is deprecated and will be removed in v1.0.0."
+          @ttl_ms = ttl_ms
+        elsif ttl != :not_provided
+          warn "DEPRECATION WARNING: ttl (seconds) parameter for PartitionsCountCache is deprecated. " \
+               "Use ttl_ms (milliseconds) instead. This parameter will be removed in v1.0.0."
+          @ttl_ms = (ttl * 1000).to_i
+        elsif ttl_ms == :not_provided
+          @ttl_ms = Defaults::PARTITIONS_COUNT_CACHE_TTL_MS
+        else
+          @ttl_ms = ttl_ms
+        end
       end
 
       # Reads partition count for a topic with automatic refresh when expired
@@ -137,17 +147,17 @@ module Rdkafka
 
           if current_info.nil?
             # Create new entry
-            @counts[topic] = [monotonic_now, new_count]
+            @counts[topic] = [monotonic_now_ms, new_count]
           else
             current_count = current_info[1]
 
             if new_count > current_count
               # Update to higher count value
-              current_info[0] = monotonic_now
+              current_info[0] = monotonic_now_ms
               current_info[1] = new_count
             else
               # Same or lower count, update timestamp only
-              current_info[0] = monotonic_now
+              current_info[0] = monotonic_now_ms
             end
           end
         end
@@ -201,15 +211,15 @@ module Rdkafka
         return unless current_info
 
         # Update the timestamp in-place
-        current_info[0] = monotonic_now
+        current_info[0] = monotonic_now_ms
       end
 
       # Check if a timestamp has expired based on the TTL
       #
-      # @param timestamp [Float] Monotonic timestamp to check
+      # @param timestamp [Integer] Monotonic timestamp in milliseconds to check
       # @return [Boolean] true if expired, false otherwise
       def expired?(timestamp)
-        monotonic_now - timestamp > @ttl
+        monotonic_now_ms - timestamp > @ttl_ms
       end
     end
   end

data/lib/rdkafka/producer.rb

@@ -6,9 +6,6 @@ module Rdkafka
     include Helpers::Time
     include Helpers::OAuth
 
-    # Empty hash used as a default
-    EMPTY_HASH = {}.freeze
-
     # @private
     @@partitions_count_cache = PartitionsCountCache.new
 
@@ -17,10 +14,9 @@ module Rdkafka
     # then. Since the partitions count can only grow and should be same for all consumers and
     # producers, we can use a global cache as long as we ensure that updates only move up.
     #
+    # @return [Rdkafka::Producer::PartitionsCountCache]
     # @note It is critical to remember, that not all users may have statistics callbacks enabled,
     #   hence we should not make assumption that this cache is always updated from the stats.
-    #
-    # @return [Rdkafka::Producer::PartitionsCountCache]
     def self.partitions_count_cache
       @@partitions_count_cache
     end
@@ -30,6 +26,9 @@ module Rdkafka
       @@partitions_count_cache = partitions_count_cache
     end
 
+    # Empty hash used as a default
+    EMPTY_HASH = {}.freeze
+
     private_constant :EMPTY_HASH
 
     # Raised when there was a critical issue when invoking rd_kafka_topic_new
@@ -64,12 +63,13 @@ module Rdkafka
     end
 
     # Sets alternative set of configuration details that can be set per topic
-    # @note It is not allowed to re-set the same topic config twice because of the underlying
-    #   librdkafka caching
+    #
     # @param topic [String] The topic name
     # @param config [Hash] config we want to use per topic basis
     # @param config_hash [Integer] hash of the config. We expect it here instead of computing it,
     #   because it is already computed during the retrieval attempt in the `#produce` flow.
+    # @note It is not allowed to re-set the same topic config twice because of the underlying
+    #   librdkafka caching
     def set_topic_config(topic, config, config_hash)
       # Ensure lock on topic reference just in case
       @native_kafka.with_inner do |inner|
@@ -80,25 +80,25 @@ module Rdkafka
 
         # If config is empty, we create an empty reference that will be used with defaults
         rd_topic_config = if config.empty?
-                            nil
-                          else
-                            Rdkafka::Bindings.rd_kafka_topic_conf_new.tap do |topic_config|
-                              config.each do |key, value|
-                                error_buffer = FFI::MemoryPointer.new(:char, 256)
-                                result = Rdkafka::Bindings.rd_kafka_topic_conf_set(
-                                  topic_config,
-                                  key.to_s,
-                                  value.to_s,
-                                  error_buffer,
-                                  256
-                                )
-
-                                unless result == :config_ok
-                                  raise Config::ConfigError.new(error_buffer.read_string)
-                                end
-                              end
-                            end
-                          end
+          nil
+        else
+          Rdkafka::Bindings.rd_kafka_topic_conf_new.tap do |topic_config|
+            config.each do |key, value|
+              error_buffer = FFI::MemoryPointer.new(:char, 256)
+              result = Rdkafka::Bindings.rd_kafka_topic_conf_set(
+                topic_config,
+                key.to_s,
+                value.to_s,
+                error_buffer,
+                256
+              )
+
+              unless result == :config_ok
+                raise Config::ConfigError.new(error_buffer.read_string)
+              end
+            end
+          end
+        end
 
         topic_handle = Bindings.rd_kafka_topic_new(inner, topic, rd_topic_config)
 
@@ -122,11 +122,30 @@ module Rdkafka
       end
     end
 
+    # Enable IO event notifications for fiber scheduler integration
+    # When delivery confirmations arrive, librdkafka will write to your FD
+    #
+    # @param fd [Integer] file descriptor to signal (from IO.pipe or eventfd)
+    # @param payload [String] data to write to fd (default: "\x01")
+    # @return [nil]
+    # @raise [ClosedInnerError] when the producer is closed
+    def enable_queue_io_events(fd, payload = "\x01")
+      @native_kafka.enable_main_queue_io_events(fd, payload)
+    end
+
+    # Enable IO event notifications for background events
+    # @param fd [Integer] file descriptor to signal (from IO.pipe or eventfd)
+    # @param payload [String] data to write to fd (default: "\x01")
+    # @return [nil]
+    # @raise [ClosedInnerError] when the producer is closed
+    def enable_background_queue_io_events(fd, payload = "\x01")
+      @native_kafka.enable_background_queue_io_events(fd, payload)
+    end
+
     # Set a callback that will be called every time a message is successfully produced.
     # The callback is called with a {DeliveryReport} and {DeliveryHandle}
     #
-    # @param callback [Proc, #call] The callback
-    #
+    # @param callback [Proc, #call] callable object to handle delivery reports
     # @return [nil]
     def delivery_callback=(callback)
       raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call)
@@ -168,7 +187,7 @@ module Rdkafka
     #   should be no other errors.
     #
     # @note For `timed_out` we do not raise an error to keep it backwards compatible
-    def flush(timeout_ms=5_000)
+    def flush(timeout_ms = Defaults::PRODUCER_FLUSH_TIMEOUT_MS)
       closed_producer_check(__method__)
 
       code = nil
@@ -208,11 +227,76 @@ module Rdkafka
       code.zero? || raise(Rdkafka::RdkafkaError.new(code))
 
       # Wait for the purge to affect everything
-      sleep(0.001) until flush(100)
+      sleep(Defaults::PRODUCER_PURGE_SLEEP_INTERVAL_MS / 1000.0) until flush(Defaults::PRODUCER_PURGE_FLUSH_TIMEOUT_MS)
 
       true
     end
 
+    # Returns the number of messages and requests waiting to be sent to the broker as well as
+    # delivery reports queued for the application.
+    #
+    # This provides visibility into the producer's internal queue depth, useful for:
+    # - Monitoring producer backpressure
+    # - Implementing custom flow control
+    # - Debugging message delivery issues
+    # - Graceful shutdown logic (wait until queue is empty)
+    #
+    # @return [Integer] the number of messages in the queue
+    # @raise [Rdkafka::ClosedProducerError] if called on a closed producer
+    #
+    # @note This method is thread-safe as it uses the @native_kafka.with_inner synchronization
+    #
+    # @example
+    #   producer.queue_size #=> 42
+    def queue_size
+      closed_producer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_outq_len(inner)
+      end
+    end
+
+    alias_method :queue_length, :queue_size
+
+    # Polls for events in a non-blocking loop, yielding the count after each iteration.
+    #
+    # This method processes delivery callbacks in a single GVL/mutex session, which is more
+    # efficient than repeated individual polls. It uses non-blocking polls internally
+    # (no GVL release between polls).
+    #
+    # Yields the count of events processed after each poll iteration, allowing the caller
+    # to implement timeout or other termination logic by returning `:stop`.
+    #
+    # @yield [count] Called after each poll iteration
+    # @yieldparam count [Integer] Number of events processed in this iteration
+    # @yieldreturn [Symbol, Object] Return `:stop` to break the loop, any other value continues
+    # @return [nil]
+    # @raise [Rdkafka::ClosedProducerError] if called on a closed producer
+    #
+    # @note This method holds the inner lock until the queue is empty or `:stop` is returned.
+    #   Other producer operations (produce, close, etc.) will wait until this method returns.
+    # @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
+    #
+    # @example Drain all pending callbacks
+    #   producer.events_poll_nb_each { |_count| }
+    #
+    # @example With timeout control
+    #   deadline = monotonic_now + timeout_ms
+    #   producer.events_poll_nb_each do |_count|
+    #     :stop if monotonic_now >= deadline
+    #   end
+    def events_poll_nb_each
+      closed_producer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        loop do
+          count = Rdkafka::Bindings.rd_kafka_poll_nb(inner, 0)
+          break if count.zero?
+          break if yield(count) == :stop
+        end
+      end
+    end
+
     # Partition count for a given topic.
     #
     # @param topic [String] The topic name.
@@ -237,13 +321,13 @@ module Rdkafka
           topic_metadata = ::Rdkafka::Metadata.new(inner, topic).topics&.first
         end
 
-        topic_metadata ? topic_metadata[:partition_count] : -1
+        topic_metadata ? topic_metadata[:partition_count] : Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
       end
     rescue Rdkafka::RdkafkaError => e
       # If the topic does not exist, it will be created or if not allowed another error will be
-      # raised. We here return -1 so this can happen without early error happening on metadata
-      # discovery.
-      return -1 if e.code == :unknown_topic_or_part
+      # raised. We here return RD_KAFKA_PARTITION_UA so this can happen without early error
+      # happening on metadata discovery.
+      return Rdkafka::Bindings::RD_KAFKA_PARTITION_UA if e.code == :unknown_topic_or_part
 
       raise(e)
     end
@@ -254,14 +338,15 @@ module Rdkafka
     # When a timestamp is provided this is used instead of the auto-generated timestamp.
     #
     # @param topic [String] The topic to produce to
-    # @param payload [String,nil] The message's payload
-    # @param key [String, nil] The message's key
-    # @param partition [Integer,nil] Optional partition to produce to
+    # @param payload [String, nil]
+    # @param key [String, nil]
+    # @param partition [Integer, nil] Optional partition to produce to
     # @param partition_key [String, nil] Optional partition key based on which partition assignment can happen
-    # @param timestamp [Time,Integer,nil] Optional timestamp of this message. Integer timestamp is in milliseconds since Jan 1 1970.
-    # @param headers [Hash<String,String|Array<String>>] Optional message headers. Values can be either a single string or an array of strings to support duplicate headers per KIP-82
+    # @param timestamp [Time, Integer, nil] Optional timestamp of this message. Integer timestamp is in milliseconds since Jan 1 1970.
+    # @param headers [Hash{String => String, Array<String>}] Optional message headers. Values can be either a single string or an array of strings to support duplicate headers per KIP-82
     # @param label [Object, nil] a label that can be assigned when producing a message that will be part of the delivery handle and the delivery report
     # @param topic_config [Hash] topic config for given message dispatch. Allows to send messages to topics with different configuration
+    # @param partitioner [String] name of the partitioner to use
     #
     # @return [DeliveryHandle] Delivery handle that can be used to wait for the result of producing this message
     #
@@ -284,17 +369,17 @@ module Rdkafka
 
       # Get payload length
       payload_size = if payload.nil?
-                       0
-                     else
-                       payload.bytesize
-                     end
+        0
+      else
+        payload.bytesize
+      end
 
       # Get key length
       key_size = if key.nil?
-                   0
-                 else
-                   key.bytesize
-                 end
+        0
+      else
+        key.bytesize
+      end
 
       topic_config_hash = topic_config.hash
 
@@ -311,36 +396,39 @@ module Rdkafka
         selected_partitioner = @topics_configs.dig(topic, topic_config_hash, :partitioner) || partitioner
 
         # If the topic is not present, set to -1
-        partition = Rdkafka::Bindings.partitioner(
-          topic_ref,
-          partition_key,
-          partition_count,
-          selected_partitioner) if partition_count.positive?
+        if partition_count.positive?
+          partition = Rdkafka::Bindings.partitioner(
+            topic_ref,
+            partition_key,
+            partition_count,
+            selected_partitioner
+          )
+        end
       end
 
-      # If partition is nil, use -1 to let librdafka set the partition randomly or
+      # If partition is nil, use RD_KAFKA_PARTITION_UA to let librdafka set the partition randomly or
       # based on the key when present.
-      partition ||= -1
+      partition ||= Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
 
       # If timestamp is nil use 0 and let Kafka set one. If an integer or time
       # use it.
       raw_timestamp = if timestamp.nil?
-                        0
-                      elsif timestamp.is_a?(Integer)
-                        timestamp
-                      elsif timestamp.is_a?(Time)
-                        (timestamp.to_i * 1000) + (timestamp.usec / 1000)
-                      else
-                        raise TypeError.new("Timestamp has to be nil, an Integer or a Time")
-                      end
+        0
+      elsif timestamp.is_a?(Integer)
+        timestamp
+      elsif timestamp.is_a?(Time)
+        (timestamp.to_i * 1000) + (timestamp.usec / 1000)
+      else
+        raise TypeError.new("Timestamp has to be nil, an Integer or a Time")
+      end
 
       delivery_handle = DeliveryHandle.new
       delivery_handle.label = label
       delivery_handle.topic = topic
       delivery_handle[:pending] = true
-      delivery_handle[:response] = -1
-      delivery_handle[:partition] = -1
-      delivery_handle[:offset] = -1
+      delivery_handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
+      delivery_handle[:partition] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
+      delivery_handle[:offset] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
       DeliveryHandle.register(delivery_handle)
 
       args = [
@@ -350,29 +438,27 @@ module Rdkafka
         :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_KEY, :buffer_in, key, :size_t, key_size,
         :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_PARTITION, :int32, partition,
         :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_TIMESTAMP, :int64, raw_timestamp,
-        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_OPAQUE, :pointer, delivery_handle,
+        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_OPAQUE, :pointer, delivery_handle
       ]
 
-      if headers
-        headers.each do |key0, value0|
-          key = key0.to_s
-          if value0.is_a?(Array)
-            # Handle array of values per KIP-82
-            value0.each do |value|
-              value = value.to_s
-              args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_HEADER
-              args << :string << key
-              args << :pointer << value
-              args << :size_t << value.bytesize
-            end
-          else
-            # Handle single value
-            value = value0.to_s
+      headers&.each do |key0, value0|
+        key = key0.to_s
+        if value0.is_a?(Array)
+          # Handle array of values per KIP-82
+          value0.each do |value|
+            value = value.to_s
             args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_HEADER
             args << :string << key
             args << :pointer << value
             args << :size_t << value.bytesize
           end
+        else
+          # Handle single value
+          value = value0.to_s
+          args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_HEADER
+          args << :string << key
+          args << :pointer << value
+          args << :size_t << value.bytesize
         end
       end
 
@@ -387,7 +473,7 @@ module Rdkafka
       end
 
       # Raise error if the produce call was not successful
-      if response != 0
+      if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
         DeliveryHandle.remove(delivery_handle.to_ptr.address)
         raise RdkafkaError.new(response)
       end

data/lib/rdkafka/version.rb

@@ -1,7 +1,10 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.22.2"
-  LIBRDKAFKA_VERSION = "2.8.0"
-  LIBRDKAFKA_SOURCE_SHA256 = "5bd1c46f63265f31c6bfcedcde78703f77d28238eadf23821c2b43fc30be3e25"
+  # Current rdkafka-ruby gem version
+  VERSION = "0.27.0"
+  # Target librdkafka version to be used
+  LIBRDKAFKA_VERSION = "2.14.0"
+  # SHA256 hash of the librdkafka source tarball for verification
+  LIBRDKAFKA_SOURCE_SHA256 = "c05c03ef00a13a8463fac3e8918c04843c416f11ced58c889d806a88ca92cf99"
 end

data/lib/rdkafka.rb

@@ -8,6 +8,7 @@ require "json"
 require "rdkafka/version"
 require "rdkafka/helpers/time"
 require "rdkafka/helpers/oauth"
+require "rdkafka/defaults"
 require "rdkafka/abstract_handle"
 require "rdkafka/admin"
 require "rdkafka/admin/create_topic_handle"
@@ -28,6 +29,8 @@ require "rdkafka/admin/describe_configs_handle"
 require "rdkafka/admin/describe_configs_report"
 require "rdkafka/admin/incremental_alter_configs_handle"
 require "rdkafka/admin/incremental_alter_configs_report"
+require "rdkafka/admin/list_offsets_handle"
+require "rdkafka/admin/list_offsets_report"
 require "rdkafka/admin/acl_binding_result"
 require "rdkafka/admin/config_binding_result"
 require "rdkafka/admin/config_resource_binding_result"