rdkafka 0.25.0-aarch64-linux-gnu → 0.26.0-aarch64-linux-gnu

data/lib/rdkafka/consumer/message.rb

@@ -53,13 +53,11 @@ module Rdkafka
         # Set timestamp
         raw_timestamp = Rdkafka::Bindings.rd_kafka_message_timestamp(native_message, nil)
         @timestamp = if raw_timestamp && raw_timestamp > -1
-                       # Calculate seconds and microseconds
-                       seconds = raw_timestamp / 1000
-                       milliseconds = (raw_timestamp - seconds * 1000) * 1000
-                       Time.at(seconds, milliseconds)
-                     else
-                       nil
-                     end
+          # Calculate seconds and microseconds
+          seconds = raw_timestamp / 1000
+          milliseconds = (raw_timestamp - seconds * 1000) * 1000
+          Time.at(seconds, milliseconds)
+        end
 
         @headers = Headers.from_native(native_message)
       end
@@ -84,7 +82,6 @@ module Rdkafka
           string
         end
       end
-
     end
   end
 end

data/lib/rdkafka/consumer/partition.rb

@@ -47,8 +47,8 @@ module Rdkafka
       # @return [Boolean]
       def ==(other)
         self.class == other.class &&
-          self.partition == other.partition &&
-          self.offset == other.offset
+          partition == other.partition &&
+          offset == other.offset
       end
     end
   end

data/lib/rdkafka/consumer/topic_partition_list.rb

@@ -9,7 +9,7 @@ module Rdkafka
       # @param data [Hash{String => nil,Partition}] The topic and partition data or nil to create an empty list
       #
       # @return [TopicPartitionList]
-      def initialize(data=nil)
+      def initialize(data = nil)
         @data = data || {}
       end
 
@@ -18,10 +18,10 @@ module Rdkafka
       def count
         i = 0
         @data.each do |_topic, partitions|
-          if partitions
-            i += partitions.count
+          i += if partitions
+            partitions.count
           else
-            i+= 1
+            1
           end
         end
         i
@@ -49,7 +49,7 @@ module Rdkafka
       #
       # @example Add a topic with all topics up to a count
       #   tpl.add_topic("topic", 9)
-      def add_topic(topic, partitions=nil)
+      def add_topic(topic, partitions = nil)
         if partitions.nil?
           @data[topic.to_s] = nil
         else
@@ -90,7 +90,7 @@ module Rdkafka
       # @param other [TopicPartitionList] object to compare with
       # @return [Boolean]
       def ==(other)
-        self.to_h == other.to_h
+        to_h == other.to_h
       end
 
       # Create a new topic partition list based of a native one.
@@ -114,10 +114,10 @@ module Rdkafka
           else
             partitions = data[elem[:topic]] || []
             offset = if elem[:offset] == Rdkafka::Bindings::RD_KAFKA_OFFSET_INVALID
-                       nil
-                     else
-                       elem[:offset]
-                     end
+              nil
+            else
+              elem[:offset]
+            end
             partition = Partition.new(elem[:partition], offset, elem[:err])
             partitions.push(partition)
             data[elem[:topic]] = partitions

data/lib/rdkafka/consumer.rb

@@ -19,6 +19,9 @@ module Rdkafka
     # @param native_kafka [NativeKafka] wrapper around the native Kafka consumer handle
     def initialize(native_kafka)
       @native_kafka = native_kafka
+
+      # Makes sure, that native kafka gets closed before it gets GCed by Ruby
+      ObjectSpace.define_finalizer(self, native_kafka.finalizer)
     end
 
     # Starts the native Kafka polling thread and kicks off the init polling
@@ -34,10 +37,142 @@ module Rdkafka
       end
     end
 
-    # @return [Proc] finalizer proc for closing the consumer
-    # @private
-    def finalizer
-      ->(_) { close }
+    # Enable IO event notifications for fiber scheduler integration
+    # When the consumer queue has messages, librdkafka will write to your FD
+    #
+    # @param fd [Integer] file descriptor to signal (from IO.pipe or eventfd)
+    # @param payload [String] data to write to fd (default: "\x01")
+    # @return [nil]
+    # @raise [ClosedInnerError] when the consumer is closed
+    #
+    # @example Using with fiber scheduler
+    #   consumer = config.consumer
+    #   consumer.subscribe("topic")
+    #
+    #   # Create notification FD
+    #   signal_r, signal_w = IO.pipe
+    #
+    #   # Enable librdkafka to signal when messages arrive
+    #   consumer.enable_queue_io_events(signal_w.fileno)
+    #
+    #   # Monitor with select/poll
+    #   loop do
+    #     readable, = IO.select([signal_r], nil, nil, timeout)
+    #     if readable
+    #       signal_r.read_nonblock(1024) rescue nil  # Drain signal
+    #       while msg = consumer.poll(0)
+    #         process(msg)
+    #       end
+    #     end
+    #   end
+    def enable_queue_io_events(fd, payload = "\x01")
+      @native_kafka.enable_main_queue_io_events(fd, payload)
+    end
+
+    # Enable IO event notifications for background events
+    # @param fd [Integer] file descriptor to signal (from IO.pipe or eventfd)
+    # @param payload [String] data to write to fd (default: "\x01")
+    # @return [nil]
+    # @raise [ClosedInnerError] when the consumer is closed
+    def enable_background_queue_io_events(fd, payload = "\x01")
+      @native_kafka.enable_background_queue_io_events(fd, payload)
+    end
+
+    # Polls for events in a non-blocking loop, yielding the count after each iteration.
+    #
+    # This method processes events (stats, errors, etc.) in a single GVL/mutex session,
+    # which is more efficient than repeated individual polls. It uses non-blocking polls
+    # internally (no GVL release between polls).
+    #
+    # Yields the count of events processed after each poll iteration, allowing the caller
+    # to implement timeout or other termination logic by returning `:stop`.
+    #
+    # @yield [count] Called after each poll iteration
+    # @yieldparam count [Integer] Number of events processed in this iteration
+    # @yieldreturn [Symbol, Object] Return `:stop` to break the loop, any other value continues
+    # @return [nil]
+    # @raise [Rdkafka::ClosedConsumerError] if called on a closed consumer
+    #
+    # @note This method holds the inner lock until the queue is empty or `:stop` is returned.
+    #   Other consumer operations will wait until this method returns.
+    # @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
+    # @note Do NOT use this if `consumer_poll_set` was set to `true`
+    #
+    # @example Drain all pending events
+    #   consumer.events_poll_nb_each { |_count| }
+    #
+    # @example With timeout control
+    #   deadline = monotonic_now + timeout_ms
+    #   consumer.events_poll_nb_each do |_count|
+    #     :stop if monotonic_now >= deadline
+    #   end
+    def events_poll_nb_each
+      closed_consumer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        loop do
+          count = Rdkafka::Bindings.rd_kafka_poll_nb(inner, 0)
+          break if count.zero?
+          break if yield(count) == :stop
+        end
+      end
+    end
+
+    # Polls for messages in a non-blocking loop, yielding each message to the caller.
+    #
+    # This method processes messages in a single GVL/mutex session until the queue is empty
+    # or the caller returns `:stop`. It handles the message pointer lifecycle internally,
+    # ensuring proper cleanup via `rd_kafka_message_destroy`.
+    #
+    # @yield [message] Called for each message received
+    # @yieldparam message [Consumer::Message] The received message
+    # @yieldreturn [Symbol, Object] Return `:stop` to break the loop, any other value continues
+    # @return [nil]
+    # @raise [Rdkafka::ClosedConsumerError] if called on a closed consumer
+    # @raise [Rdkafka::RdkafkaError] if a Kafka error occurs while polling
+    #
+    # @note This method uses `rd_kafka_consumer_poll` to fetch messages, unlike
+    #   `events_poll_nb_each` which uses `rd_kafka_poll` for event callbacks (delivery reports,
+    #   statistics, etc.). For consumers, use this method to receive messages and
+    #   `events_poll_nb_each` for processing background events.
+    # @note This method holds the inner lock for the duration. Other consumer operations
+    #   will wait until this method returns.
+    # @note Timeout/max_messages logic should be implemented by the caller
+    #
+    # @example Process messages until queue is empty
+    #   consumer.poll_nb_each do |message|
+    #     process(message)
+    #   end
+    #
+    # @example Process with early termination
+    #   count = 0
+    #   consumer.poll_nb_each do |message|
+    #     process(message)
+    #     count += 1
+    #     :stop if count >= 10
+    #   end
+    def poll_nb_each
+      closed_consumer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        loop do
+          message_ptr = Rdkafka::Bindings.rd_kafka_consumer_poll_nb(inner, 0)
+          break if message_ptr.null?
+
+          begin
+            native_message = Rdkafka::Bindings::Message.new(message_ptr)
+
+            if native_message[:err] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+              raise Rdkafka::RdkafkaError.new(native_message[:err])
+            end
+
+            result = yield Consumer::Message.new(native_message)
+            break if result == :stop
+          ensure
+            Rdkafka::Bindings.rd_kafka_message_destroy(message_ptr)
+          end
+        end
+      end
     end
 
     # Close this consumer
@@ -78,7 +213,7 @@ module Rdkafka
         Rdkafka::Bindings.rd_kafka_subscribe(inner, tpl)
       end
       if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
-        raise Rdkafka::RdkafkaError.new(response, "Error subscribing to '#{topics.join(', ')}'")
+        raise Rdkafka::RdkafkaError.new(response, "Error subscribing to '#{topics.join(", ")}'")
       end
     ensure
       Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) unless tpl.nil?
@@ -228,7 +363,7 @@ module Rdkafka
         end
       end
     ensure
-      ptr.free unless ptr.nil?
+      ptr&.free
     end
 
     # @return [Boolean] true if our current assignment has been lost involuntarily.
@@ -281,7 +416,7 @@ module Rdkafka
     # @return [TopicPartitionList]
     #
     # @raise [RdkafkaError] When getting the positions fails.
-    def position(list=nil)
+    def position(list = nil)
       if list.nil?
         list = assignment
       elsif !list.is_a?(TopicPartitionList)
@@ -321,17 +456,17 @@ module Rdkafka
           partition,
           low,
           high,
-          timeout_ms,
+          timeout_ms
         )
       end
       if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
         raise Rdkafka::RdkafkaError.new(response, "Error querying watermark offsets for partition #{partition} of #{topic}")
       end
 
-      return low.read_array_of_int64(1).first, high.read_array_of_int64(1).first
+      [low.read_array_of_int64(1).first, high.read_array_of_int64(1).first]
     ensure
-      low.free   unless low.nil?
-      high.free  unless high.nil?
+      low&.free
+      high&.free
     end
 
     # Calculate the consumer lag per partition for the provided topic partition list.
@@ -510,14 +645,14 @@ module Rdkafka
     # @param async [Boolean] Whether to commit async or wait for the commit to finish
     # @return [nil]
     # @raise [RdkafkaError] When committing fails
-    def commit(list=nil, async=false)
+    def commit(list = nil, async = false)
       closed_consumer_check(__method__)
 
       if !list.nil? && !list.is_a?(TopicPartitionList)
         raise TypeError.new("list has to be nil or a TopicPartitionList")
       end
 
-      tpl = list ? list.to_native_tpl : nil
+      tpl = list&.to_native_tpl
 
       begin
         response = @native_kafka.with_inner do |inner|
@@ -561,6 +696,47 @@ module Rdkafka
       end
     end
 
+    # Poll for the next message without releasing the GVL (Global VM Lock).
+    #
+    # This is more efficient than regular polling for non-blocking poll(0) calls,
+    # particularly useful in fiber scheduler contexts where GVL release/reacquire
+    # overhead is wasteful since we don't expect to wait.
+    #
+    # @param timeout_ms [Integer] Timeout of this poll (default: 0 for non-blocking)
+    # @return [Message, nil] A message or nil if there was no new message within the timeout
+    # @raise [RdkafkaError] When polling fails
+    #
+    # @example Using with fiber scheduler
+    #   # After receiving IO notification that messages are available
+    #   while msg = consumer.poll_nb
+    #     process(msg)
+    #   end
+    def poll_nb(timeout_ms = 0)
+      closed_consumer_check(__method__)
+
+      message_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_consumer_poll_nb(inner, timeout_ms)
+      end
+
+      if message_ptr.null?
+        nil
+      else
+        # Create struct wrapper
+        native_message = Rdkafka::Bindings::Message.new(message_ptr)
+        # Raise error if needed
+        if native_message[:err] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+          raise Rdkafka::RdkafkaError.new(native_message[:err])
+        end
+        # Create a message to pass out
+        Rdkafka::Consumer::Message.new(native_message)
+      end
+    ensure
+      # Clean up rdkafka message if there is one
+      if message_ptr && !message_ptr.null?
+        Rdkafka::Bindings.rd_kafka_message_destroy(message_ptr)
+      end
+    end
+
     # Polls the main rdkafka queue (not the consumer one). Do **NOT** use it if `consumer_poll_set`
     #   was set to `true`.
     #
@@ -587,6 +763,22 @@ module Rdkafka
       end
     end
 
+    # Polls the main rdkafka queue without releasing the GVL (Global VM Lock).
+    #
+    # This is more efficient than regular events_poll for non-blocking poll(0) calls,
+    # particularly useful in fiber scheduler contexts where GVL release/reacquire
+    # overhead is wasteful since we don't expect to wait.
+    #
+    # @param timeout_ms [Integer] poll timeout (default: 0 for non-blocking)
+    # @return [Integer] the number of events served
+    #
+    # @see #events_poll for more details on when to use this method
+    def events_poll_nb(timeout_ms = 0)
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_poll_nb(inner, timeout_ms)
+      end
+    end
+
     # Poll for new messages and yield for each received one. Iteration
     # will end when the consumer is closed.
     #
@@ -602,12 +794,10 @@ module Rdkafka
         message = poll(timeout_ms)
         if message
           yield(message)
+        elsif closed?
+          break
         else
-          if closed?
-            break
-          else
-            next
-          end
+          next
         end
       end
     end

data/lib/rdkafka/error.rb

@@ -18,15 +18,21 @@ module Rdkafka
     # @return [String]
     attr_reader :broker_message
 
+    # The name of the rdkafka instance that generated this error
+    # @return [String, nil]
+    attr_reader :instance_name
+
     # @private
     # @param response [Integer] the raw error response code from librdkafka
     # @param message_prefix [String, nil] optional prefix for error messages
     # @param broker_message [String, nil] optional error message from the broker
-    def initialize(response, message_prefix=nil, broker_message: nil)
+    # @param instance_name [String, nil] optional name of the rdkafka instance
+    def initialize(response, message_prefix = nil, broker_message: nil, instance_name: nil)
       raise TypeError.new("Response has to be an integer") unless response.is_a? Integer
       @rdkafka_response = response
       @message_prefix = message_prefix
       @broker_message = broker_message
+      @instance_name = instance_name
     end
 
     # This error's code, for example `:partition_eof`, `:msg_size_too_large`.
@@ -34,7 +40,7 @@ module Rdkafka
     def code
       code = Rdkafka::Bindings.rd_kafka_err2name(@rdkafka_response).downcase
       if code[0] == "_"
-        code[1..-1].to_sym
+        code[1..].to_sym
       else
         code.to_sym
       end
@@ -44,11 +50,16 @@ module Rdkafka
     # @return [String]
     def to_s
       message_prefix_part = if message_prefix
-                       "#{message_prefix} - "
-                     else
-                       ''
-                     end
-      "#{message_prefix_part}#{Rdkafka::Bindings.rd_kafka_err2str(@rdkafka_response)} (#{code})"
+        "#{message_prefix} - "
+      else
+        ""
+      end
+      instance_name_part = if instance_name
+        " [#{instance_name}]"
+      else
+        ""
+      end
+      "#{message_prefix_part}#{Rdkafka::Bindings.rd_kafka_err2str(@rdkafka_response)} (#{code})#{instance_name_part}"
     end
 
     # Whether this error indicates the partition is EOF.
@@ -58,10 +69,10 @@ module Rdkafka
     end
 
     # Error comparison
-    # @param another_error [Object] object to compare with
+    # @param other [Object] object to compare with
     # @return [Boolean]
-    def ==(another_error)
-       another_error.is_a?(self.class) && (self.to_s == another_error.to_s)
+    def ==(other)
+      other.is_a?(self.class) && (to_s == other.to_s)
     end
   end
 
@@ -74,7 +85,7 @@ module Rdkafka
     # @param response [Integer] the raw error response code from librdkafka
     # @param topic_partition_list [TopicPartitionList] the topic partition list with error info
     # @param message_prefix [String, nil] optional prefix for error messages
-    def initialize(response, topic_partition_list, message_prefix=nil)
+    def initialize(response, topic_partition_list, message_prefix = nil)
       super(response, message_prefix)
       @topic_partition_list = topic_partition_list
     end
@@ -84,7 +95,7 @@ module Rdkafka
   class ClosedConsumerError < BaseError
     # @param method [Symbol] the method that was called
     def initialize(method)
-      super("Illegal call to #{method.to_s} on a closed consumer")
+      super("Illegal call to #{method} on a closed consumer")
     end
   end
 
@@ -92,7 +103,7 @@ module Rdkafka
   class ClosedProducerError < BaseError
     # @param method [Symbol] the method that was called
     def initialize(method)
-      super("Illegal call to #{method.to_s} on a closed producer")
+      super("Illegal call to #{method} on a closed producer")
     end
   end
 
@@ -100,7 +111,7 @@ module Rdkafka
   class ClosedAdminError < BaseError
     # @param method [Symbol] the method that was called
     def initialize(method)
-      super("Illegal call to #{method.to_s} on a closed admin")
+      super("Illegal call to #{method} on a closed admin")
     end
   end
 

data/lib/rdkafka/helpers/oauth.rb

@@ -2,7 +2,6 @@ module Rdkafka
   module Helpers
     # OAuth helper methods for setting and refreshing SASL/OAUTHBEARER tokens
     module OAuth
-
       # Set the OAuthBearer token
       #
       # @param token [String] the mandatory token value to set, often (but not necessarily) a JWS compact serialization as per https://tools.ietf.org/html/rfc7515#section-3.1.

data/lib/rdkafka/helpers/time.rb

@@ -9,6 +9,11 @@ module Rdkafka
       def monotonic_now
         ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       end
+
+      # @return [Integer] current monotonic time in milliseconds
+      def monotonic_now_ms
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :millisecond)
+      end
     end
   end
 end

data/lib/rdkafka/metadata.rb

@@ -100,40 +100,40 @@ module Rdkafka
     # FFI struct for rd_kafka_metadata_t
     class Metadata < CustomFFIStruct
       layout :brokers_count, :int,
-             :brokers_metadata, :pointer,
-             :topics_count, :int,
-             :topics_metadata, :pointer,
-             :broker_id, :int32,
-             :broker_name, :string
+        :brokers_metadata, :pointer,
+        :topics_count, :int,
+        :topics_metadata, :pointer,
+        :broker_id, :int32,
+        :broker_name, :string
     end
 
     # @private
     # FFI struct for rd_kafka_metadata_broker_t
     class BrokerMetadata < CustomFFIStruct
       layout :broker_id, :int32,
-             :broker_name, :string,
-             :broker_port, :int
+        :broker_name, :string,
+        :broker_port, :int
     end
 
     # @private
     # FFI struct for rd_kafka_metadata_topic_t
     class TopicMetadata < CustomFFIStruct
       layout :topic_name, :string,
-             :partition_count, :int,
-             :partitions_metadata, :pointer,
-             :rd_kafka_resp_err, :int
+        :partition_count, :int,
+        :partitions_metadata, :pointer,
+        :rd_kafka_resp_err, :int
     end
 
     # @private
     # FFI struct for rd_kafka_metadata_partition_t
     class PartitionMetadata < CustomFFIStruct
       layout :partition_id, :int32,
-             :rd_kafka_resp_err, :int,
-             :leader, :int32,
-             :replica_count, :int,
-             :replicas, :pointer,
-             :in_sync_replica_brokers, :int,
-             :isrs, :pointer
+        :rd_kafka_resp_err, :int,
+        :leader, :int32,
+        :replica_count, :int,
+        :replicas, :pointer,
+        :in_sync_replica_brokers, :int,
+        :isrs, :pointer
     end
   end
 end

data/lib/rdkafka/native_kafka.rb

@@ -70,7 +70,7 @@ module Rdkafka
             end
           end
 
-          @polling_thread.name = "rdkafka.native_kafka##{Rdkafka::Bindings.rd_kafka_name(@inner).gsub('rdkafka', '')}"
+          @polling_thread.name = "rdkafka.native_kafka##{Rdkafka::Bindings.rd_kafka_name(@inner).gsub("rdkafka", "")}"
           @polling_thread.abort_on_exception = true
           @polling_thread[:closing] = false
         end
@@ -121,11 +121,72 @@ module Rdkafka
       @closing || @inner.nil?
     end
 
+    # Enable IO event notifications on the main queue
+    # Librdkafka will write to your FD when the queue transitions from empty to non-empty
+    #
+    # @note This method is incompatible with background polling threads.
+    #   If background polling is enabled, use manual polling instead (e.g., consumer.poll)
+    #
+    # @param fd [Integer] your file descriptor (from IO.pipe or eventfd)
+    # @param payload [String] data to write to fd when queue has data (default: "\x01")
+    # @return [nil]
+    # @raise [ClosedInnerError] when the handle is closed
+    # @raise [RuntimeError] when background polling thread is active
+    #
+    # @example
+    #   # Create your own signaling FD
+    #   signal_r, signal_w = IO.pipe
+    #   native_kafka.enable_main_queue_io_events(signal_w.fileno)
+    #
+    #   # Monitor it with select
+    #   readable, = IO.select([signal_r], nil, nil, timeout)
+    #   if readable
+    #     consumer.poll(0)  # Get messages
+    #   end
+    def enable_main_queue_io_events(fd, payload = "\x01")
+      if @run_polling_thread
+        raise "Cannot enable IO events while background polling thread is active. " \
+          "Either disable background polling by setting run_polling_thread: false, " \
+          "or use manual polling with consumer.poll() instead of the FD API."
+      end
+
+      with_inner do |inner|
+        queue_ptr = Bindings.rd_kafka_queue_get_main(inner)
+        Bindings.rd_kafka_queue_io_event_enable(queue_ptr, fd, payload, payload.bytesize)
+        Bindings.rd_kafka_queue_destroy(queue_ptr)
+      end
+    end
+
+    # Enable IO event notifications on the background queue
+    # Librdkafka will write to your FD when the background queue transitions from empty to non-empty
+    #
+    # @note This method is incompatible with background polling threads.
+    #   If background polling is enabled, use manual polling instead (e.g., consumer.poll)
+    #
+    # @param fd [Integer] your file descriptor (from IO.pipe or eventfd)
+    # @param payload [String] data to write to fd when queue has data (default: "\x01")
+    # @return [nil]
+    # @raise [ClosedInnerError] when the handle is closed
+    # @raise [RuntimeError] when background polling thread is active
+    def enable_background_queue_io_events(fd, payload = "\x01")
+      if @run_polling_thread
+        raise "Cannot enable IO events while background polling thread is active. " \
+          "Either disable background polling by setting run_polling_thread: false, " \
+          "or use manual polling with consumer.poll() instead of the FD API."
+      end
+
+      with_inner do |inner|
+        queue_ptr = Bindings.rd_kafka_queue_get_background(inner)
+        Bindings.rd_kafka_queue_io_event_enable(queue_ptr, fd, payload, payload.bytesize)
+        Bindings.rd_kafka_queue_destroy(queue_ptr)
+      end
+    end
+
     # Closes the native Kafka handle and cleans up resources
     # @param object_id [Integer, nil] optional object ID (unused, for finalizer compatibility)
     # @yield optional block to execute before destroying the handle
     # @return [nil]
-    def close(object_id=nil)
+    def close(object_id = nil)
       return if closed?
 
       synchronize do