rdkafka 0.24.2-aarch64-linux-gnu → 0.25.1-aarch64-linux-gnu

data/lib/rdkafka/consumer/topic_partition_list.rb

@@ -9,7 +9,7 @@ module Rdkafka
       # @param data [Hash{String => nil,Partition}] The topic and partition data or nil to create an empty list
       #
       # @return [TopicPartitionList]
-      def initialize(data=nil)
+      def initialize(data = nil)
         @data = data || {}
       end
 
@@ -18,10 +18,10 @@ module Rdkafka
       def count
         i = 0
         @data.each do |_topic, partitions|
-          if partitions
-            i += partitions.count
+          i += if partitions
+            partitions.count
           else
-            i+= 1
+            1
           end
         end
         i
@@ -49,15 +49,14 @@ module Rdkafka
       #
       # @example Add a topic with all topics up to a count
       #   tpl.add_topic("topic", 9)
-      #
-      def add_topic(topic, partitions=nil)
+      def add_topic(topic, partitions = nil)
         if partitions.nil?
           @data[topic.to_s] = nil
         else
           if partitions.is_a? Integer
             partitions = (0..partitions - 1)
           end
-          @data[topic.to_s] = partitions.map { |p| Partition.new(p, nil, 0) }
+          @data[topic.to_s] = partitions.map { |p| Partition.new(p, nil, Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR) }
         end
       end
 
@@ -65,10 +64,8 @@ module Rdkafka
       # Calling this method multiple times for the same topic will overwrite the previous configuraton.
       #
       # @param topic [String] The topic's name
-      # @param partitions_with_offsets [Hash<Integer, Integer>] The topic's partitions and offsets
-      # @param partitions_with_offsets [Array<Consumer::Partition>] The topic's partitions with offsets
-      #   and metadata (if any)
-      #
+      # @param partitions_with_offsets [Hash{Integer => Integer}, Array<Consumer::Partition>] The topic's
+      #   partitions and offsets (Hash) or partitions with offsets and metadata (Array)
       # @return [nil]
       def add_topic_and_partitions_with_offsets(topic, partitions_with_offsets)
         @data[topic.to_s] = partitions_with_offsets.map do |p, o|
@@ -89,8 +86,11 @@ module Rdkafka
         "<TopicPartitionList: #{to_h}>"
       end
 
+      # Check equality with another TopicPartitionList
+      # @param other [TopicPartitionList] object to compare with
+      # @return [Boolean]
       def ==(other)
-        self.to_h == other.to_h
+        to_h == other.to_h
       end
 
       # Create a new topic partition list based of a native one.
@@ -109,15 +109,15 @@ module Rdkafka
         native_tpl[:cnt].times do |i|
           ptr = native_tpl[:elems] + (i * Rdkafka::Bindings::TopicPartition.size)
           elem = Rdkafka::Bindings::TopicPartition.new(ptr)
-          if elem[:partition] == -1
+          if elem[:partition] == Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
             data[elem[:topic]] = nil
           else
             partitions = data[elem[:topic]] || []
             offset = if elem[:offset] == Rdkafka::Bindings::RD_KAFKA_OFFSET_INVALID
-                       nil
-                     else
-                       elem[:offset]
-                     end
+              nil
+            else
+              elem[:offset]
+            end
             partition = Partition.new(elem[:partition], offset, elem[:err])
             partitions.push(partition)
             data[elem[:topic]] = partitions
@@ -161,7 +161,7 @@ module Rdkafka
             Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
               tpl,
               topic,
-              -1
+              Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
             )
           end
         end

data/lib/rdkafka/consumer.rb

@@ -16,6 +16,7 @@ module Rdkafka
     include Helpers::OAuth
 
     # @private
+    # @param native_kafka [NativeKafka] wrapper around the native Kafka consumer handle
     def initialize(native_kafka)
       @native_kafka = native_kafka
     end
@@ -33,6 +34,146 @@ module Rdkafka
       end
     end
 
+    # Enable IO event notifications for fiber scheduler integration
+    # When the consumer queue has messages, librdkafka will write to your FD
+    #
+    # @param fd [Integer] file descriptor to signal (from IO.pipe or eventfd)
+    # @param payload [String] data to write to fd (default: "\x01")
+    # @return [nil]
+    # @raise [ClosedInnerError] when the consumer is closed
+    #
+    # @example Using with fiber scheduler
+    #   consumer = config.consumer
+    #   consumer.subscribe("topic")
+    #
+    #   # Create notification FD
+    #   signal_r, signal_w = IO.pipe
+    #
+    #   # Enable librdkafka to signal when messages arrive
+    #   consumer.enable_queue_io_events(signal_w.fileno)
+    #
+    #   # Monitor with select/poll
+    #   loop do
+    #     readable, = IO.select([signal_r], nil, nil, timeout)
+    #     if readable
+    #       signal_r.read_nonblock(1024) rescue nil  # Drain signal
+    #       while msg = consumer.poll(0)
+    #         process(msg)
+    #       end
+    #     end
+    #   end
+    def enable_queue_io_events(fd, payload = "\x01")
+      @native_kafka.enable_main_queue_io_events(fd, payload)
+    end
+
+    # Enable IO event notifications for background events
+    # @param fd [Integer] file descriptor to signal (from IO.pipe or eventfd)
+    # @param payload [String] data to write to fd (default: "\x01")
+    # @return [nil]
+    # @raise [ClosedInnerError] when the consumer is closed
+    def enable_background_queue_io_events(fd, payload = "\x01")
+      @native_kafka.enable_background_queue_io_events(fd, payload)
+    end
+
+    # Polls for events in a non-blocking loop, yielding the count after each iteration.
+    #
+    # This method processes events (stats, errors, etc.) in a single GVL/mutex session,
+    # which is more efficient than repeated individual polls. It uses non-blocking polls
+    # internally (no GVL release between polls).
+    #
+    # Yields the count of events processed after each poll iteration, allowing the caller
+    # to implement timeout or other termination logic by returning `:stop`.
+    #
+    # @yield [count] Called after each poll iteration
+    # @yieldparam count [Integer] Number of events processed in this iteration
+    # @yieldreturn [Symbol, Object] Return `:stop` to break the loop, any other value continues
+    # @return [nil]
+    # @raise [Rdkafka::ClosedConsumerError] if called on a closed consumer
+    #
+    # @note This method holds the inner lock until the queue is empty or `:stop` is returned.
+    #   Other consumer operations will wait until this method returns.
+    # @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
+    # @note Do NOT use this if `consumer_poll_set` was set to `true`
+    #
+    # @example Drain all pending events
+    #   consumer.events_poll_nb_each { |_count| }
+    #
+    # @example With timeout control
+    #   deadline = monotonic_now + timeout_ms
+    #   consumer.events_poll_nb_each do |_count|
+    #     :stop if monotonic_now >= deadline
+    #   end
+    def events_poll_nb_each
+      closed_consumer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        loop do
+          count = Rdkafka::Bindings.rd_kafka_poll_nb(inner, 0)
+          break if count.zero?
+          break if yield(count) == :stop
+        end
+      end
+    end
+
+    # Polls for messages in a non-blocking loop, yielding each message to the caller.
+    #
+    # This method processes messages in a single GVL/mutex session until the queue is empty
+    # or the caller returns `:stop`. It handles the message pointer lifecycle internally,
+    # ensuring proper cleanup via `rd_kafka_message_destroy`.
+    #
+    # @yield [message] Called for each message received
+    # @yieldparam message [Consumer::Message] The received message
+    # @yieldreturn [Symbol, Object] Return `:stop` to break the loop, any other value continues
+    # @return [nil]
+    # @raise [Rdkafka::ClosedConsumerError] if called on a closed consumer
+    # @raise [Rdkafka::RdkafkaError] if a Kafka error occurs while polling
+    #
+    # @note This method uses `rd_kafka_consumer_poll` to fetch messages, unlike
+    #   `events_poll_nb_each` which uses `rd_kafka_poll` for event callbacks (delivery reports,
+    #   statistics, etc.). For consumers, use this method to receive messages and
+    #   `events_poll_nb_each` for processing background events.
+    # @note This method holds the inner lock for the duration. Other consumer operations
+    #   will wait until this method returns.
+    # @note Timeout/max_messages logic should be implemented by the caller
+    #
+    # @example Process messages until queue is empty
+    #   consumer.poll_nb_each do |message|
+    #     process(message)
+    #   end
+    #
+    # @example Process with early termination
+    #   count = 0
+    #   consumer.poll_nb_each do |message|
+    #     process(message)
+    #     count += 1
+    #     :stop if count >= 10
+    #   end
+    def poll_nb_each
+      closed_consumer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        loop do
+          message_ptr = Rdkafka::Bindings.rd_kafka_consumer_poll_nb(inner, 0)
+          break if message_ptr.null?
+
+          begin
+            native_message = Rdkafka::Bindings::Message.new(message_ptr)
+
+            if native_message[:err] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+              raise Rdkafka::RdkafkaError.new(native_message[:err])
+            end
+
+            result = yield Consumer::Message.new(native_message)
+            break if result == :stop
+          ensure
+            Rdkafka::Bindings.rd_kafka_message_destroy(message_ptr)
+          end
+        end
+      end
+    end
+
+    # @return [Proc] finalizer proc for closing the consumer
+    # @private
     def finalizer
       ->(_) { close }
     end
@@ -67,15 +208,15 @@ module Rdkafka
       tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(topics.length)
 
       topics.each do |topic|
-        Rdkafka::Bindings.rd_kafka_topic_partition_list_add(tpl, topic, -1)
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_add(tpl, topic, Rdkafka::Bindings::RD_KAFKA_PARTITION_UA)
       end
 
       # Subscribe to topic partition list and check this was successful
       response = @native_kafka.with_inner do |inner|
         Rdkafka::Bindings.rd_kafka_subscribe(inner, tpl)
       end
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response, "Error subscribing to '#{topics.join(', ')}'")
+      if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+        raise Rdkafka::RdkafkaError.new(response, "Error subscribing to '#{topics.join(", ")}'")
       end
     ensure
       Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) unless tpl.nil?
@@ -91,7 +232,7 @@ module Rdkafka
       response = @native_kafka.with_inner do |inner|
         Rdkafka::Bindings.rd_kafka_unsubscribe(inner)
       end
-      if response != 0
+      if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
         raise Rdkafka::RdkafkaError.new(response)
       end
     end
@@ -115,7 +256,7 @@ module Rdkafka
           Rdkafka::Bindings.rd_kafka_pause_partitions(inner, tpl)
         end
 
-        if response != 0
+        if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
           list = TopicPartitionList.from_native_tpl(tpl)
           raise Rdkafka::RdkafkaTopicPartitionListError.new(response, list, "Error pausing '#{list.to_h}'")
         end
@@ -142,7 +283,7 @@ module Rdkafka
         response = @native_kafka.with_inner do |inner|
           Rdkafka::Bindings.rd_kafka_resume_partitions(inner, tpl)
         end
-        if response != 0
+        if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
           raise Rdkafka::RdkafkaError.new(response, "Error resume '#{list.to_h}'")
         end
       ensure
@@ -162,7 +303,7 @@ module Rdkafka
         Rdkafka::Bindings.rd_kafka_subscription(inner, ptr)
       end
 
-      if response != 0
+      if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
         raise Rdkafka::RdkafkaError.new(response)
       end
 
@@ -192,7 +333,7 @@ module Rdkafka
         response = @native_kafka.with_inner do |inner|
           Rdkafka::Bindings.rd_kafka_assign(inner, tpl)
         end
-        if response != 0
+        if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
           raise Rdkafka::RdkafkaError.new(response, "Error assigning '#{list.to_h}'")
         end
       ensure
@@ -211,7 +352,7 @@ module Rdkafka
       response = @native_kafka.with_inner do |inner|
         Rdkafka::Bindings.rd_kafka_assignment(inner, ptr)
       end
-      if response != 0
+      if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
         raise Rdkafka::RdkafkaError.new(response)
       end
 
@@ -225,7 +366,7 @@ module Rdkafka
         end
       end
     ensure
-      ptr.free unless ptr.nil?
+      ptr&.free
     end
 
     # @return [Boolean] true if our current assignment has been lost involuntarily.
@@ -246,7 +387,7 @@ module Rdkafka
     # @param timeout_ms [Integer] The timeout for fetching this information.
     # @return [TopicPartitionList]
     # @raise [RdkafkaError] When getting the committed positions fails.
-    def committed(list=nil, timeout_ms=2_000)
+    def committed(list = nil, timeout_ms = Defaults::CONSUMER_COMMITTED_TIMEOUT_MS)
       closed_consumer_check(__method__)
 
       if list.nil?
@@ -261,7 +402,7 @@ module Rdkafka
         response = @native_kafka.with_inner do |inner|
           Rdkafka::Bindings.rd_kafka_committed(inner, tpl, timeout_ms)
         end
-        if response != 0
+        if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
           raise Rdkafka::RdkafkaError.new(response)
         end
         TopicPartitionList.from_native_tpl(tpl)
@@ -278,7 +419,7 @@ module Rdkafka
     # @return [TopicPartitionList]
     #
     # @raise [RdkafkaError] When getting the positions fails.
-    def position(list=nil)
+    def position(list = nil)
       if list.nil?
         list = assignment
       elsif !list.is_a?(TopicPartitionList)
@@ -291,7 +432,7 @@ module Rdkafka
         Rdkafka::Bindings.rd_kafka_position(inner, tpl)
       end
 
-      if response != 0
+      if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
         raise Rdkafka::RdkafkaError.new(response)
       end
 
@@ -305,7 +446,7 @@ module Rdkafka
     # @param timeout_ms [Integer] The timeout for querying the broker
     # @return [Integer] The low and high watermark
     # @raise [RdkafkaError] When querying the broker fails.
-    def query_watermark_offsets(topic, partition, timeout_ms=1000)
+    def query_watermark_offsets(topic, partition, timeout_ms = Defaults::CONSUMER_QUERY_WATERMARK_TIMEOUT_MS)
       closed_consumer_check(__method__)
 
       low = FFI::MemoryPointer.new(:int64, 1)
@@ -318,17 +459,17 @@ module Rdkafka
           partition,
           low,
           high,
-          timeout_ms,
+          timeout_ms
         )
       end
-      if response != 0
+      if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
         raise Rdkafka::RdkafkaError.new(response, "Error querying watermark offsets for partition #{partition} of #{topic}")
       end
 
-      return low.read_array_of_int64(1).first, high.read_array_of_int64(1).first
+      [low.read_array_of_int64(1).first, high.read_array_of_int64(1).first]
     ensure
-      low.free   unless low.nil?
-      high.free  unless high.nil?
+      low&.free
+      high&.free
     end
 
     # Calculate the consumer lag per partition for the provided topic partition list.
@@ -338,10 +479,10 @@ module Rdkafka
     #
     # @param topic_partition_list [TopicPartitionList] The list to calculate lag for.
     # @param watermark_timeout_ms [Integer] The timeout for each query watermark call.
-    # @return [Hash<String, Hash<Integer, Integer>>] A hash containing all topics with the lag
+    # @return [Hash{String => Hash{Integer => Integer}}] A hash containing all topics with the lag
     #   per partition
     # @raise [RdkafkaError] When querying the broker fails.
-    def lag(topic_partition_list, watermark_timeout_ms=1000)
+    def lag(topic_partition_list, watermark_timeout_ms = Defaults::CONSUMER_LAG_TIMEOUT_MS)
       out = {}
 
       topic_partition_list.to_h.each do |topic, partitions|
@@ -409,7 +550,7 @@ module Rdkafka
         )
       end
 
-      if response != 0
+      if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
         raise Rdkafka::RdkafkaError.new(response)
       end
     ensure
@@ -451,9 +592,9 @@ module Rdkafka
         native_topic,
         partition,
         offset,
-        0 # timeout
+        Defaults::CONSUMER_SEEK_TIMEOUT_MS
       )
-      if response != 0
+      if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
         raise Rdkafka::RdkafkaError.new(response)
       end
     ensure
@@ -465,11 +606,10 @@ module Rdkafka
     # Lookup offset for the given partitions by timestamp.
     #
     # @param list [TopicPartitionList] The TopicPartitionList with timestamps instead of offsets
-    #
+    # @param timeout_ms [Integer] timeout in milliseconds for the operation
     # @return [TopicPartitionList]
-    #
     # @raise [RdKafkaError] When the OffsetForTimes lookup fails
-    def offsets_for_times(list, timeout_ms = 1000)
+    def offsets_for_times(list, timeout_ms = Defaults::CONSUMER_OFFSETS_FOR_TIMES_TIMEOUT_MS)
       closed_consumer_check(__method__)
 
       if !list.is_a?(TopicPartitionList)
@@ -486,7 +626,7 @@ module Rdkafka
         )
       end
 
-      if response != 0
+      if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
         raise Rdkafka::RdkafkaError.new(response)
       end
 
@@ -508,20 +648,20 @@ module Rdkafka
     # @param async [Boolean] Whether to commit async or wait for the commit to finish
     # @return [nil]
     # @raise [RdkafkaError] When committing fails
-    def commit(list=nil, async=false)
+    def commit(list = nil, async = false)
       closed_consumer_check(__method__)
 
       if !list.nil? && !list.is_a?(TopicPartitionList)
         raise TypeError.new("list has to be nil or a TopicPartitionList")
       end
 
-      tpl = list ? list.to_native_tpl : nil
+      tpl = list&.to_native_tpl
 
       begin
         response = @native_kafka.with_inner do |inner|
           Rdkafka::Bindings.rd_kafka_commit(inner, tpl, async)
         end
-        if response != 0
+        if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
           raise Rdkafka::RdkafkaError.new(response)
         end
       ensure
@@ -546,7 +686,48 @@ module Rdkafka
         # Create struct wrapper
         native_message = Rdkafka::Bindings::Message.new(message_ptr)
         # Raise error if needed
-        if native_message[:err] != 0
+        if native_message[:err] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+          raise Rdkafka::RdkafkaError.new(native_message[:err])
+        end
+        # Create a message to pass out
+        Rdkafka::Consumer::Message.new(native_message)
+      end
+    ensure
+      # Clean up rdkafka message if there is one
+      if message_ptr && !message_ptr.null?
+        Rdkafka::Bindings.rd_kafka_message_destroy(message_ptr)
+      end
+    end
+
+    # Poll for the next message without releasing the GVL (Global VM Lock).
+    #
+    # This is more efficient than regular polling for non-blocking poll(0) calls,
+    # particularly useful in fiber scheduler contexts where GVL release/reacquire
+    # overhead is wasteful since we don't expect to wait.
+    #
+    # @param timeout_ms [Integer] Timeout of this poll (default: 0 for non-blocking)
+    # @return [Message, nil] A message or nil if there was no new message within the timeout
+    # @raise [RdkafkaError] When polling fails
+    #
+    # @example Using with fiber scheduler
+    #   # After receiving IO notification that messages are available
+    #   while msg = consumer.poll_nb
+    #     process(msg)
+    #   end
+    def poll_nb(timeout_ms = 0)
+      closed_consumer_check(__method__)
+
+      message_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_consumer_poll_nb(inner, timeout_ms)
+      end
+
+      if message_ptr.null?
+        nil
+      else
+        # Create struct wrapper
+        native_message = Rdkafka::Bindings::Message.new(message_ptr)
+        # Raise error if needed
+        if native_message[:err] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
           raise Rdkafka::RdkafkaError.new(native_message[:err])
         end
         # Create a message to pass out
@@ -579,37 +760,58 @@ module Rdkafka
     # @note This method technically should be called `#poll` and the current `#poll` should be
     #   called `#consumer_poll` though we keep the current naming convention to make it backward
     #   compatible.
-    def events_poll(timeout_ms = 0)
+    def events_poll(timeout_ms = Defaults::CONSUMER_EVENTS_POLL_TIMEOUT_MS)
       @native_kafka.with_inner do |inner|
         Rdkafka::Bindings.rd_kafka_poll(inner, timeout_ms)
       end
     end
 
+    # Polls the main rdkafka queue without releasing the GVL (Global VM Lock).
+    #
+    # This is more efficient than regular events_poll for non-blocking poll(0) calls,
+    # particularly useful in fiber scheduler contexts where GVL release/reacquire
+    # overhead is wasteful since we don't expect to wait.
+    #
+    # @param timeout_ms [Integer] poll timeout (default: 0 for non-blocking)
+    # @return [Integer] the number of events served
+    #
+    # @see #events_poll for more details on when to use this method
+    def events_poll_nb(timeout_ms = 0)
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_poll_nb(inner, timeout_ms)
+      end
+    end
+
     # Poll for new messages and yield for each received one. Iteration
     # will end when the consumer is closed.
     #
     # If `enable.partition.eof` is turned on in the config this will raise an error when an eof is
     # reached, so you probably want to disable that when using this method of iteration.
     #
+    # @param timeout_ms [Integer] Timeout for each poll iteration
     # @yieldparam message [Message] Received message
     # @return [nil]
     # @raise [RdkafkaError] When polling fails
-    def each
+    def each(timeout_ms: Defaults::CONSUMER_POLL_TIMEOUT_MS)
       loop do
-        message = poll(250)
+        message = poll(timeout_ms)
         if message
           yield(message)
+        elsif closed?
+          break
         else
-          if closed?
-            break
-          else
-            next
-          end
+          next
         end
       end
     end
 
-    # Deprecated. Please read the error message for more details.
+    # @deprecated This method has been removed due to data consistency concerns
+    # @param max_items [Integer] unused
+    # @param bytes_threshold [Numeric] unused
+    # @param timeout_ms [Integer] unused
+    # @param yield_on_error [Boolean] unused
+    # @param block [Proc] unused block
+    # @raise [NotImplementedError] Always raises as this method is no longer supported
     def each_batch(max_items: 100, bytes_threshold: Float::INFINITY, timeout_ms: 250, yield_on_error: false, &block)
       raise NotImplementedError, <<~ERROR
         `each_batch` has been removed due to data consistency concerns.
@@ -646,6 +848,9 @@ module Rdkafka
 
     private
 
+    # Checks if the consumer is closed and raises an error if so
+    # @param method [Symbol] name of the calling method for error context
+    # @raise [ClosedConsumerError] when the consumer is closed
     def closed_consumer_check(method)
       raise Rdkafka::ClosedConsumerError.new(method) if closed?
     end