rdkafka 0.22.2 → 0.29.0

data/lib/rdkafka/producer.rb

@@ -5,9 +5,7 @@ module Rdkafka
   class Producer
     include Helpers::Time
     include Helpers::OAuth
-
-    # Empty hash used as a default
-    EMPTY_HASH = {}.freeze
+    include Helpers::Metadata
 
     # @private
     @@partitions_count_cache = PartitionsCountCache.new
@@ -17,10 +15,9 @@ module Rdkafka
     # then. Since the partitions count can only grow and should be same for all consumers and
     # producers, we can use a global cache as long as we ensure that updates only move up.
     #
+    # @return [Rdkafka::Producer::PartitionsCountCache]
     # @note It is critical to remember, that not all users may have statistics callbacks enabled,
     #   hence we should not make assumption that this cache is always updated from the stats.
-    #
-    # @return [Rdkafka::Producer::PartitionsCountCache]
     def self.partitions_count_cache
       @@partitions_count_cache
     end
@@ -30,6 +27,9 @@ module Rdkafka
       @@partitions_count_cache = partitions_count_cache
     end
 
+    # Empty hash used as a default
+    EMPTY_HASH = {}.freeze
+
     private_constant :EMPTY_HASH
 
     # Raised when there was a critical issue when invoking rd_kafka_topic_new
@@ -64,12 +64,13 @@ module Rdkafka
     end
 
     # Sets alternative set of configuration details that can be set per topic
-    # @note It is not allowed to re-set the same topic config twice because of the underlying
-    #   librdkafka caching
+    #
     # @param topic [String] The topic name
     # @param config [Hash] config we want to use per topic basis
     # @param config_hash [Integer] hash of the config. We expect it here instead of computing it,
     #   because it is already computed during the retrieval attempt in the `#produce` flow.
+    # @note It is not allowed to re-set the same topic config twice because of the underlying
+    #   librdkafka caching
     def set_topic_config(topic, config, config_hash)
       # Ensure lock on topic reference just in case
       @native_kafka.with_inner do |inner|
@@ -80,25 +81,31 @@ module Rdkafka
 
         # If config is empty, we create an empty reference that will be used with defaults
         rd_topic_config = if config.empty?
-                            nil
-                          else
-                            Rdkafka::Bindings.rd_kafka_topic_conf_new.tap do |topic_config|
-                              config.each do |key, value|
-                                error_buffer = FFI::MemoryPointer.new(:char, 256)
-                                result = Rdkafka::Bindings.rd_kafka_topic_conf_set(
-                                  topic_config,
-                                  key.to_s,
-                                  value.to_s,
-                                  error_buffer,
-                                  256
-                                )
-
-                                unless result == :config_ok
-                                  raise Config::ConfigError.new(error_buffer.read_string)
-                                end
-                              end
-                            end
-                          end
+          nil
+        else
+          Rdkafka::Bindings.rd_kafka_topic_conf_new.tap do |topic_config|
+            config.each do |key, value|
+              error_buffer = FFI::MemoryPointer.new(:char, 256)
+              result = Rdkafka::Bindings.rd_kafka_topic_conf_set(
+                topic_config,
+                key.to_s,
+                value.to_s,
+                error_buffer,
+                256
+              )
+
+              unless result == :config_ok
+                # rd_kafka_topic_new has not been called yet, so librdkafka has not taken
+                # ownership of the config. Destroy it ourselves before raising, otherwise the
+                # partially built rd_kafka_topic_conf_t leaks. (On the rd_kafka_topic_new path
+                # librdkafka frees the conf on both success and failure, so we never destroy it
+                # there.)
+                Rdkafka::Bindings.rd_kafka_topic_conf_destroy(topic_config)
+                raise Config::ConfigError.new(error_buffer.read_string)
+              end
+            end
+          end
+        end
 
         topic_handle = Bindings.rd_kafka_topic_new(inner, topic, rd_topic_config)
 
@@ -122,11 +129,30 @@ module Rdkafka
       end
     end
 
+    # Enable IO event notifications for fiber scheduler integration
+    # When delivery confirmations arrive, librdkafka will write to your FD
+    #
+    # @param fd [Integer] file descriptor to signal (from IO.pipe or eventfd)
+    # @param payload [String] data to write to fd (default: "\x01")
+    # @return [nil]
+    # @raise [ClosedInnerError] when the producer is closed
+    def enable_queue_io_events(fd, payload = "\x01")
+      @native_kafka.enable_main_queue_io_events(fd, payload)
+    end
+
+    # Enable IO event notifications for background events
+    # @param fd [Integer] file descriptor to signal (from IO.pipe or eventfd)
+    # @param payload [String] data to write to fd (default: "\x01")
+    # @return [nil]
+    # @raise [ClosedInnerError] when the producer is closed
+    def enable_background_queue_io_events(fd, payload = "\x01")
+      @native_kafka.enable_background_queue_io_events(fd, payload)
+    end
+
     # Set a callback that will be called every time a message is successfully produced.
     # The callback is called with a {DeliveryReport} and {DeliveryHandle}
     #
-    # @param callback [Proc, #call] The callback
-    #
+    # @param callback [Proc, #call] callable object to handle delivery reports
     # @return [nil]
     def delivery_callback=(callback)
       raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call)
@@ -168,7 +194,7 @@ module Rdkafka
     #   should be no other errors.
     #
     # @note For `timed_out` we do not raise an error to keep it backwards compatible
-    def flush(timeout_ms=5_000)
+    def flush(timeout_ms = Defaults::PRODUCER_FLUSH_TIMEOUT_MS)
       closed_producer_check(__method__)
 
       code = nil
@@ -208,11 +234,76 @@ module Rdkafka
       code.zero? || raise(Rdkafka::RdkafkaError.new(code))
 
       # Wait for the purge to affect everything
-      sleep(0.001) until flush(100)
+      sleep(Defaults::PRODUCER_PURGE_SLEEP_INTERVAL_MS / 1000.0) until flush(Defaults::PRODUCER_PURGE_FLUSH_TIMEOUT_MS)
 
       true
     end
 
+    # Returns the number of messages and requests waiting to be sent to the broker as well as
+    # delivery reports queued for the application.
+    #
+    # This provides visibility into the producer's internal queue depth, useful for:
+    # - Monitoring producer backpressure
+    # - Implementing custom flow control
+    # - Debugging message delivery issues
+    # - Graceful shutdown logic (wait until queue is empty)
+    #
+    # @return [Integer] the number of messages in the queue
+    # @raise [Rdkafka::ClosedProducerError] if called on a closed producer
+    #
+    # @note This method is thread-safe as it uses the @native_kafka.with_inner synchronization
+    #
+    # @example
+    #   producer.queue_size #=> 42
+    def queue_size
+      closed_producer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_outq_len(inner)
+      end
+    end
+
+    alias_method :queue_length, :queue_size
+
+    # Polls for events in a non-blocking loop, yielding the count after each iteration.
+    #
+    # This method processes delivery callbacks in a single GVL/mutex session, which is more
+    # efficient than repeated individual polls. It uses non-blocking polls internally
+    # (no GVL release between polls).
+    #
+    # Yields the count of events processed after each poll iteration, allowing the caller
+    # to implement timeout or other termination logic by returning `:stop`.
+    #
+    # @yield [count] Called after each poll iteration
+    # @yieldparam count [Integer] Number of events processed in this iteration
+    # @yieldreturn [Symbol, Object] Return `:stop` to break the loop, any other value continues
+    # @return [nil]
+    # @raise [Rdkafka::ClosedProducerError] if called on a closed producer
+    #
+    # @note This method holds the inner lock until the queue is empty or `:stop` is returned.
+    #   Other producer operations (produce, close, etc.) will wait until this method returns.
+    # @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
+    #
+    # @example Drain all pending callbacks
+    #   producer.events_poll_nb_each { |_count| }
+    #
+    # @example With timeout control
+    #   deadline = monotonic_now + timeout_ms
+    #   producer.events_poll_nb_each do |_count|
+    #     :stop if monotonic_now >= deadline
+    #   end
+    def events_poll_nb_each
+      closed_producer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        loop do
+          count = Rdkafka::Bindings.rd_kafka_poll_nb(inner, 0)
+          break if count.zero?
+          break if yield(count) == :stop
+        end
+      end
+    end
+
     # Partition count for a given topic.
     #
     # @param topic [String] The topic name.
@@ -237,15 +328,16 @@ module Rdkafka
           topic_metadata = ::Rdkafka::Metadata.new(inner, topic).topics&.first
         end
 
-        topic_metadata ? topic_metadata[:partition_count] : -1
-      end
-    rescue Rdkafka::RdkafkaError => e
-      # If the topic does not exist, it will be created or if not allowed another error will be
-      # raised. We here return -1 so this can happen without early error happening on metadata
-      # discovery.
-      return -1 if e.code == :unknown_topic_or_part
+        topic_metadata ? topic_metadata[:partition_count] : Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
+      rescue Rdkafka::RdkafkaError => e
+        # A missing topic is an expected, cacheable outcome (the topic may be auto-created on
+        # produce, or simply not exist yet). Returning RD_KAFKA_PARTITION_UA from inside the block
+        # caches it, so we don't re-run a blocking metadata query on every produce(partition_key:)
+        # to that topic. Anything else is a real error and is re-raised.
+        raise(e) unless e.code == :unknown_topic_or_part
 
-      raise(e)
+        Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
+      end
     end
 
     # Produces a message to a Kafka topic. The message is added to rdkafka's queue, call {DeliveryHandle#wait wait} on the returned delivery handle to make sure it is delivered.
@@ -254,14 +346,15 @@ module Rdkafka
     # When a timestamp is provided this is used instead of the auto-generated timestamp.
     #
     # @param topic [String] The topic to produce to
-    # @param payload [String,nil] The message's payload
-    # @param key [String, nil] The message's key
-    # @param partition [Integer,nil] Optional partition to produce to
+    # @param payload [String, nil]
+    # @param key [String, nil]
+    # @param partition [Integer, nil] Optional partition to produce to
     # @param partition_key [String, nil] Optional partition key based on which partition assignment can happen
-    # @param timestamp [Time,Integer,nil] Optional timestamp of this message. Integer timestamp is in milliseconds since Jan 1 1970.
-    # @param headers [Hash<String,String|Array<String>>] Optional message headers. Values can be either a single string or an array of strings to support duplicate headers per KIP-82
+    # @param timestamp [Time, Integer, nil] Optional timestamp of this message. Integer timestamp is in milliseconds since Jan 1 1970.
+    # @param headers [Hash{String => String, Array<String>}] Optional message headers. Values can be either a single string or an array of strings to support duplicate headers per KIP-82
     # @param label [Object, nil] a label that can be assigned when producing a message that will be part of the delivery handle and the delivery report
     # @param topic_config [Hash] topic config for given message dispatch. Allows to send messages to topics with different configuration
+    # @param partitioner [String] name of the partitioner to use
     #
     # @return [DeliveryHandle] Delivery handle that can be used to wait for the result of producing this message
     #
@@ -284,17 +377,17 @@ module Rdkafka
 
       # Get payload length
       payload_size = if payload.nil?
-                       0
-                     else
-                       payload.bytesize
-                     end
+        0
+      else
+        payload.bytesize
+      end
 
       # Get key length
       key_size = if key.nil?
-                   0
-                 else
-                   key.bytesize
-                 end
+        0
+      else
+        key.bytesize
+      end
 
       topic_config_hash = topic_config.hash
 
@@ -311,50 +404,52 @@ module Rdkafka
         selected_partitioner = @topics_configs.dig(topic, topic_config_hash, :partitioner) || partitioner
 
         # If the topic is not present, set to -1
-        partition = Rdkafka::Bindings.partitioner(
-          topic_ref,
-          partition_key,
-          partition_count,
-          selected_partitioner) if partition_count.positive?
+        if partition_count.positive?
+          partition = Rdkafka::Bindings.partitioner(
+            topic_ref,
+            partition_key,
+            partition_count,
+            selected_partitioner
+          )
+        end
       end
 
-      # If partition is nil, use -1 to let librdafka set the partition randomly or
+      # If partition is nil, use RD_KAFKA_PARTITION_UA to let librdafka set the partition randomly or
       # based on the key when present.
-      partition ||= -1
+      partition ||= Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
 
-      # If timestamp is nil use 0 and let Kafka set one. If an integer or time
-      # use it.
+      # If timestamp is nil use 0 and let Kafka set one. If an integer or time use it.
       raw_timestamp = if timestamp.nil?
-                        0
-                      elsif timestamp.is_a?(Integer)
-                        timestamp
-                      elsif timestamp.is_a?(Time)
-                        (timestamp.to_i * 1000) + (timestamp.usec / 1000)
-                      else
-                        raise TypeError.new("Timestamp has to be nil, an Integer or a Time")
-                      end
+        0
+      elsif timestamp.is_a?(Integer)
+        timestamp
+      elsif timestamp.is_a?(Time)
+        (timestamp.to_i * 1000) + (timestamp.usec / 1000)
+      else
+        raise TypeError.new("Timestamp has to be nil, an Integer or a Time")
+      end
 
       delivery_handle = DeliveryHandle.new
       delivery_handle.label = label
       delivery_handle.topic = topic
       delivery_handle[:pending] = true
-      delivery_handle[:response] = -1
-      delivery_handle[:partition] = -1
-      delivery_handle[:offset] = -1
+      delivery_handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
+      delivery_handle[:partition] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
+      delivery_handle[:offset] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
       DeliveryHandle.register(delivery_handle)
 
-      args = [
-        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_RKT, :pointer, topic_ref,
-        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_MSGFLAGS, :int, Rdkafka::Bindings::RD_KAFKA_MSG_F_COPY,
-        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_VALUE, :buffer_in, payload, :size_t, payload_size,
-        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_KEY, :buffer_in, key, :size_t, key_size,
-        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_PARTITION, :int32, partition,
-        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_TIMESTAMP, :int64, raw_timestamp,
-        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_OPAQUE, :pointer, delivery_handle,
-      ]
-
-      if headers
-        headers.each do |key0, value0|
+      begin
+        args = [
+          :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_RKT, :pointer, topic_ref,
+          :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_MSGFLAGS, :int, Rdkafka::Bindings::RD_KAFKA_MSG_F_COPY,
+          :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_VALUE, :buffer_in, payload, :size_t, payload_size,
+          :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_KEY, :buffer_in, key, :size_t, key_size,
+          :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_PARTITION, :int32, partition,
+          :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_TIMESTAMP, :int64, raw_timestamp,
+          :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_OPAQUE, :pointer, delivery_handle
+        ]
+
+        headers&.each do |key0, value0|
           key = key0.to_s
           if value0.is_a?(Array)
             # Handle array of values per KIP-82
@@ -374,22 +469,24 @@ module Rdkafka
             args << :size_t << value.bytesize
           end
         end
-      end
 
-      args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_END
+        args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_END
 
-      # Produce the message
-      response = @native_kafka.with_inner do |inner|
-        Rdkafka::Bindings.rd_kafka_producev(
-          inner,
-          *args
-        )
-      end
+        # Produce the message
+        response = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_producev(
+            inner,
+            *args
+          )
+        end
 
-      # Raise error if the produce call was not successful
-      if response != 0
+        # Raise error if the produce call was not successful
+        if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+          raise RdkafkaError.new(response)
+        end
+      rescue Exception
         DeliveryHandle.remove(delivery_handle.to_ptr.address)
-        raise RdkafkaError.new(response)
+        raise
       end
 
       delivery_handle
@@ -431,5 +528,6 @@ module Rdkafka
     def closed_producer_check(method)
       raise Rdkafka::ClosedProducerError.new(method) if closed?
     end
+    alias_method :closed_check, :closed_producer_check
   end
 end

data/lib/rdkafka/version.rb

@@ -1,7 +1,10 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.22.2"
-  LIBRDKAFKA_VERSION = "2.8.0"
-  LIBRDKAFKA_SOURCE_SHA256 = "5bd1c46f63265f31c6bfcedcde78703f77d28238eadf23821c2b43fc30be3e25"
+  # Current rdkafka-ruby gem version
+  VERSION = "0.29.0"
+  # Target librdkafka version to be used
+  LIBRDKAFKA_VERSION = "2.14.2"
+  # SHA256 hash of the librdkafka source tarball for verification
+  LIBRDKAFKA_SOURCE_SHA256 = "d7eec9c31c817fa44402f679c252dfbf97e4c338a849a25c3579a31fd127beb8"
 end

data/lib/rdkafka.rb

@@ -8,6 +8,8 @@ require "json"
 require "rdkafka/version"
 require "rdkafka/helpers/time"
 require "rdkafka/helpers/oauth"
+require "rdkafka/helpers/metadata"
+require "rdkafka/defaults"
 require "rdkafka/abstract_handle"
 require "rdkafka/admin"
 require "rdkafka/admin/create_topic_handle"
@@ -28,11 +30,24 @@ require "rdkafka/admin/describe_configs_handle"
 require "rdkafka/admin/describe_configs_report"
 require "rdkafka/admin/incremental_alter_configs_handle"
 require "rdkafka/admin/incremental_alter_configs_report"
+require "rdkafka/admin/list_offsets_handle"
+require "rdkafka/admin/list_offsets_report"
 require "rdkafka/admin/acl_binding_result"
 require "rdkafka/admin/config_binding_result"
 require "rdkafka/admin/config_resource_binding_result"
 require "rdkafka/bindings"
 require "rdkafka/callbacks"
+require "rdkafka/callbacks/base_handler"
+require "rdkafka/callbacks/create_topic_handler"
+require "rdkafka/callbacks/delete_topic_handler"
+require "rdkafka/callbacks/create_partitions_handler"
+require "rdkafka/callbacks/delete_groups_handler"
+require "rdkafka/callbacks/create_acl_handler"
+require "rdkafka/callbacks/delete_acl_handler"
+require "rdkafka/callbacks/describe_acl_handler"
+require "rdkafka/callbacks/describe_configs_handler"
+require "rdkafka/callbacks/incremental_alter_configs_handler"
+require "rdkafka/callbacks/list_offsets_handler"
 require "rdkafka/config"
 require "rdkafka/consumer"
 require "rdkafka/consumer/headers"