rdkafka 0.22.2 → 0.27.0

data/lib/rdkafka/consumer.rb

@@ -16,8 +16,12 @@ module Rdkafka
     include Helpers::OAuth
 
     # @private
+    # @param native_kafka [NativeKafka] wrapper around the native Kafka consumer handle
     def initialize(native_kafka)
       @native_kafka = native_kafka
+
+      # Makes sure, that native kafka gets closed before it gets GCed by Ruby
+      ObjectSpace.define_finalizer(self, native_kafka.finalizer)
     end
 
     # Starts the native Kafka polling thread and kicks off the init polling
@@ -33,8 +37,142 @@ module Rdkafka
       end
     end
 
-    def finalizer
-      ->(_) { close }
+    # Enable IO event notifications for fiber scheduler integration
+    # When the consumer queue has messages, librdkafka will write to your FD
+    #
+    # @param fd [Integer] file descriptor to signal (from IO.pipe or eventfd)
+    # @param payload [String] data to write to fd (default: "\x01")
+    # @return [nil]
+    # @raise [ClosedInnerError] when the consumer is closed
+    #
+    # @example Using with fiber scheduler
+    #   consumer = config.consumer
+    #   consumer.subscribe("topic")
+    #
+    #   # Create notification FD
+    #   signal_r, signal_w = IO.pipe
+    #
+    #   # Enable librdkafka to signal when messages arrive
+    #   consumer.enable_queue_io_events(signal_w.fileno)
+    #
+    #   # Monitor with select/poll
+    #   loop do
+    #     readable, = IO.select([signal_r], nil, nil, timeout)
+    #     if readable
+    #       signal_r.read_nonblock(1024) rescue nil  # Drain signal
+    #       while msg = consumer.poll(0)
+    #         process(msg)
+    #       end
+    #     end
+    #   end
+    def enable_queue_io_events(fd, payload = "\x01")
+      @native_kafka.enable_main_queue_io_events(fd, payload)
+    end
+
+    # Enable IO event notifications for background events
+    # @param fd [Integer] file descriptor to signal (from IO.pipe or eventfd)
+    # @param payload [String] data to write to fd (default: "\x01")
+    # @return [nil]
+    # @raise [ClosedInnerError] when the consumer is closed
+    def enable_background_queue_io_events(fd, payload = "\x01")
+      @native_kafka.enable_background_queue_io_events(fd, payload)
+    end
+
+    # Polls for events in a non-blocking loop, yielding the count after each iteration.
+    #
+    # This method processes events (stats, errors, etc.) in a single GVL/mutex session,
+    # which is more efficient than repeated individual polls. It uses non-blocking polls
+    # internally (no GVL release between polls).
+    #
+    # Yields the count of events processed after each poll iteration, allowing the caller
+    # to implement timeout or other termination logic by returning `:stop`.
+    #
+    # @yield [count] Called after each poll iteration
+    # @yieldparam count [Integer] Number of events processed in this iteration
+    # @yieldreturn [Symbol, Object] Return `:stop` to break the loop, any other value continues
+    # @return [nil]
+    # @raise [Rdkafka::ClosedConsumerError] if called on a closed consumer
+    #
+    # @note This method holds the inner lock until the queue is empty or `:stop` is returned.
+    #   Other consumer operations will wait until this method returns.
+    # @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
+    # @note Do NOT use this if `consumer_poll_set` was set to `true`
+    #
+    # @example Drain all pending events
+    #   consumer.events_poll_nb_each { |_count| }
+    #
+    # @example With timeout control
+    #   deadline = monotonic_now + timeout_ms
+    #   consumer.events_poll_nb_each do |_count|
+    #     :stop if monotonic_now >= deadline
+    #   end
+    def events_poll_nb_each
+      closed_consumer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        loop do
+          count = Rdkafka::Bindings.rd_kafka_poll_nb(inner, 0)
+          break if count.zero?
+          break if yield(count) == :stop
+        end
+      end
+    end
+
+    # Polls for messages in a non-blocking loop, yielding each message to the caller.
+    #
+    # This method processes messages in a single GVL/mutex session until the queue is empty
+    # or the caller returns `:stop`. It handles the message pointer lifecycle internally,
+    # ensuring proper cleanup via `rd_kafka_message_destroy`.
+    #
+    # @yield [message] Called for each message received
+    # @yieldparam message [Consumer::Message] The received message
+    # @yieldreturn [Symbol, Object] Return `:stop` to break the loop, any other value continues
+    # @return [nil]
+    # @raise [Rdkafka::ClosedConsumerError] if called on a closed consumer
+    # @raise [Rdkafka::RdkafkaError] if a Kafka error occurs while polling
+    #
+    # @note This method uses `rd_kafka_consumer_poll` to fetch messages, unlike
+    #   `events_poll_nb_each` which uses `rd_kafka_poll` for event callbacks (delivery reports,
+    #   statistics, etc.). For consumers, use this method to receive messages and
+    #   `events_poll_nb_each` for processing background events.
+    # @note This method holds the inner lock for the duration. Other consumer operations
+    #   will wait until this method returns.
+    # @note Timeout/max_messages logic should be implemented by the caller
+    #
+    # @example Process messages until queue is empty
+    #   consumer.poll_nb_each do |message|
+    #     process(message)
+    #   end
+    #
+    # @example Process with early termination
+    #   count = 0
+    #   consumer.poll_nb_each do |message|
+    #     process(message)
+    #     count += 1
+    #     :stop if count >= 10
+    #   end
+    def poll_nb_each
+      closed_consumer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        loop do
+          message_ptr = Rdkafka::Bindings.rd_kafka_consumer_poll_nb(inner, 0)
+          break if message_ptr.null?
+
+          begin
+            native_message = Rdkafka::Bindings::Message.new(message_ptr)
+
+            if native_message[:err] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+              raise Rdkafka::RdkafkaError.new(native_message[:err])
+            end
+
+            result = yield Consumer::Message.new(native_message)
+            break if result == :stop
+          ensure
+            Rdkafka::Bindings.rd_kafka_message_destroy(message_ptr)
+          end
+        end
+      end
     end
 
     # Close this consumer
@@ -45,6 +183,11 @@ module Rdkafka
 
       @native_kafka.synchronize do |inner|
         Rdkafka::Bindings.rd_kafka_consumer_close(inner)
+
+        if @consumer_queue
+          Rdkafka::Bindings.rd_kafka_queue_destroy(@consumer_queue)
+          @consumer_queue = nil
+        end
       end
 
       @native_kafka.close
@@ -67,15 +210,15 @@ module Rdkafka
       tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(topics.length)
 
       topics.each do |topic|
-        Rdkafka::Bindings.rd_kafka_topic_partition_list_add(tpl, topic, -1)
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_add(tpl, topic, Rdkafka::Bindings::RD_KAFKA_PARTITION_UA)
       end
 
       # Subscribe to topic partition list and check this was successful
       response = @native_kafka.with_inner do |inner|
         Rdkafka::Bindings.rd_kafka_subscribe(inner, tpl)
       end
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response, "Error subscribing to '#{topics.join(', ')}'")
+      if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+        raise Rdkafka::RdkafkaError.new(response, "Error subscribing to '#{topics.join(", ")}'")
       end
     ensure
       Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) unless tpl.nil?
@@ -91,7 +234,7 @@ module Rdkafka
       response = @native_kafka.with_inner do |inner|
         Rdkafka::Bindings.rd_kafka_unsubscribe(inner)
       end
-      if response != 0
+      if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
         raise Rdkafka::RdkafkaError.new(response)
       end
     end
@@ -115,7 +258,7 @@ module Rdkafka
           Rdkafka::Bindings.rd_kafka_pause_partitions(inner, tpl)
         end
 
-        if response != 0
+        if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
           list = TopicPartitionList.from_native_tpl(tpl)
           raise Rdkafka::RdkafkaTopicPartitionListError.new(response, list, "Error pausing '#{list.to_h}'")
         end
@@ -142,7 +285,7 @@ module Rdkafka
         response = @native_kafka.with_inner do |inner|
           Rdkafka::Bindings.rd_kafka_resume_partitions(inner, tpl)
         end
-        if response != 0
+        if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
           raise Rdkafka::RdkafkaError.new(response, "Error resume '#{list.to_h}'")
         end
       ensure
@@ -162,7 +305,7 @@ module Rdkafka
         Rdkafka::Bindings.rd_kafka_subscription(inner, ptr)
       end
 
-      if response != 0
+      if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
         raise Rdkafka::RdkafkaError.new(response)
       end
 
@@ -192,7 +335,7 @@ module Rdkafka
         response = @native_kafka.with_inner do |inner|
           Rdkafka::Bindings.rd_kafka_assign(inner, tpl)
         end
-        if response != 0
+        if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
           raise Rdkafka::RdkafkaError.new(response, "Error assigning '#{list.to_h}'")
         end
       ensure
@@ -211,7 +354,7 @@ module Rdkafka
       response = @native_kafka.with_inner do |inner|
         Rdkafka::Bindings.rd_kafka_assignment(inner, ptr)
       end
-      if response != 0
+      if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
         raise Rdkafka::RdkafkaError.new(response)
       end
 
@@ -225,7 +368,7 @@ module Rdkafka
         end
       end
     ensure
-      ptr.free unless ptr.nil?
+      ptr&.free
     end
 
     # @return [Boolean] true if our current assignment has been lost involuntarily.
@@ -246,7 +389,7 @@ module Rdkafka
     # @param timeout_ms [Integer] The timeout for fetching this information.
     # @return [TopicPartitionList]
     # @raise [RdkafkaError] When getting the committed positions fails.
-    def committed(list=nil, timeout_ms=2000)
+    def committed(list = nil, timeout_ms = Defaults::CONSUMER_COMMITTED_TIMEOUT_MS)
       closed_consumer_check(__method__)
 
       if list.nil?
@@ -261,7 +404,7 @@ module Rdkafka
         response = @native_kafka.with_inner do |inner|
           Rdkafka::Bindings.rd_kafka_committed(inner, tpl, timeout_ms)
         end
-        if response != 0
+        if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
           raise Rdkafka::RdkafkaError.new(response)
         end
         TopicPartitionList.from_native_tpl(tpl)
@@ -278,7 +421,7 @@ module Rdkafka
     # @return [TopicPartitionList]
     #
     # @raise [RdkafkaError] When getting the positions fails.
-    def position(list=nil)
+    def position(list = nil)
       if list.nil?
         list = assignment
       elsif !list.is_a?(TopicPartitionList)
@@ -291,11 +434,13 @@ module Rdkafka
         Rdkafka::Bindings.rd_kafka_position(inner, tpl)
       end
 
-      if response != 0
+      if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
         raise Rdkafka::RdkafkaError.new(response)
       end
 
       TopicPartitionList.from_native_tpl(tpl)
+    ensure
+      Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) if tpl
     end
 
     # Query broker for low (oldest/beginning) and high (newest/end) offsets for a partition.
@@ -305,7 +450,7 @@ module Rdkafka
     # @param timeout_ms [Integer] The timeout for querying the broker
     # @return [Integer] The low and high watermark
     # @raise [RdkafkaError] When querying the broker fails.
-    def query_watermark_offsets(topic, partition, timeout_ms=1000)
+    def query_watermark_offsets(topic, partition, timeout_ms = Defaults::CONSUMER_QUERY_WATERMARK_TIMEOUT_MS)
       closed_consumer_check(__method__)
 
       low = FFI::MemoryPointer.new(:int64, 1)
@@ -318,17 +463,17 @@ module Rdkafka
           partition,
           low,
           high,
-          timeout_ms,
+          timeout_ms
         )
       end
-      if response != 0
+      if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
         raise Rdkafka::RdkafkaError.new(response, "Error querying watermark offsets for partition #{partition} of #{topic}")
       end
 
-      return low.read_array_of_int64(1).first, high.read_array_of_int64(1).first
+      [low.read_array_of_int64(1).first, high.read_array_of_int64(1).first]
     ensure
-      low.free   unless low.nil?
-      high.free  unless high.nil?
+      low&.free
+      high&.free
     end
 
     # Calculate the consumer lag per partition for the provided topic partition list.
@@ -338,10 +483,10 @@ module Rdkafka
     #
     # @param topic_partition_list [TopicPartitionList] The list to calculate lag for.
     # @param watermark_timeout_ms [Integer] The timeout for each query watermark call.
-    # @return [Hash<String, Hash<Integer, Integer>>] A hash containing all topics with the lag
+    # @return [Hash{String => Hash{Integer => Integer}}] A hash containing all topics with the lag
     #   per partition
     # @raise [RdkafkaError] When querying the broker fails.
-    def lag(topic_partition_list, watermark_timeout_ms=1000)
+    def lag(topic_partition_list, watermark_timeout_ms = Defaults::CONSUMER_LAG_TIMEOUT_MS)
       out = {}
 
       topic_partition_list.to_h.each do |topic, partitions|
@@ -350,7 +495,7 @@ module Rdkafka
         topic_out = {}
         partitions.each do |p|
           next if p.offset.nil?
-          low, high = query_watermark_offsets(
+          _low, high = query_watermark_offsets(
             topic,
             p.partition,
             watermark_timeout_ms
@@ -409,7 +554,7 @@ module Rdkafka
         )
       end
 
-      if response != 0
+      if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
         raise Rdkafka::RdkafkaError.new(response)
       end
     ensure
@@ -451,9 +596,9 @@ module Rdkafka
         native_topic,
         partition,
         offset,
-        0 # timeout
+        Defaults::CONSUMER_SEEK_TIMEOUT_MS
       )
-      if response != 0
+      if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
         raise Rdkafka::RdkafkaError.new(response)
       end
     ensure
@@ -465,11 +610,10 @@ module Rdkafka
     # Lookup offset for the given partitions by timestamp.
     #
     # @param list [TopicPartitionList] The TopicPartitionList with timestamps instead of offsets
-    #
+    # @param timeout_ms [Integer] timeout in milliseconds for the operation
     # @return [TopicPartitionList]
-    #
     # @raise [RdKafkaError] When the OffsetForTimes lookup fails
-    def offsets_for_times(list, timeout_ms = 1000)
+    def offsets_for_times(list, timeout_ms = Defaults::CONSUMER_OFFSETS_FOR_TIMES_TIMEOUT_MS)
       closed_consumer_check(__method__)
 
       if !list.is_a?(TopicPartitionList)
@@ -486,7 +630,7 @@ module Rdkafka
         )
       end
 
-      if response != 0
+      if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
         raise Rdkafka::RdkafkaError.new(response)
       end
 
@@ -508,20 +652,20 @@ module Rdkafka
     # @param async [Boolean] Whether to commit async or wait for the commit to finish
     # @return [nil]
     # @raise [RdkafkaError] When committing fails
-    def commit(list=nil, async=false)
+    def commit(list = nil, async = false)
       closed_consumer_check(__method__)
 
       if !list.nil? && !list.is_a?(TopicPartitionList)
         raise TypeError.new("list has to be nil or a TopicPartitionList")
       end
 
-      tpl = list ? list.to_native_tpl : nil
+      tpl = list&.to_native_tpl
 
       begin
         response = @native_kafka.with_inner do |inner|
           Rdkafka::Bindings.rd_kafka_commit(inner, tpl, async)
         end
-        if response != 0
+        if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
           raise Rdkafka::RdkafkaError.new(response)
         end
       ensure
@@ -546,7 +690,48 @@ module Rdkafka
         # Create struct wrapper
         native_message = Rdkafka::Bindings::Message.new(message_ptr)
         # Raise error if needed
-        if native_message[:err] != 0
+        if native_message[:err] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+          raise Rdkafka::RdkafkaError.new(native_message[:err])
+        end
+        # Create a message to pass out
+        Rdkafka::Consumer::Message.new(native_message)
+      end
+    ensure
+      # Clean up rdkafka message if there is one
+      if message_ptr && !message_ptr.null?
+        Rdkafka::Bindings.rd_kafka_message_destroy(message_ptr)
+      end
+    end
+
+    # Poll for the next message without releasing the GVL (Global VM Lock).
+    #
+    # This is more efficient than regular polling for non-blocking poll(0) calls,
+    # particularly useful in fiber scheduler contexts where GVL release/reacquire
+    # overhead is wasteful since we don't expect to wait.
+    #
+    # @param timeout_ms [Integer] Timeout of this poll (default: 0 for non-blocking)
+    # @return [Message, nil] A message or nil if there was no new message within the timeout
+    # @raise [RdkafkaError] When polling fails
+    #
+    # @example Using with fiber scheduler
+    #   # After receiving IO notification that messages are available
+    #   while msg = consumer.poll_nb
+    #     process(msg)
+    #   end
+    def poll_nb(timeout_ms = 0)
+      closed_consumer_check(__method__)
+
+      message_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_consumer_poll_nb(inner, timeout_ms)
+      end
+
+      if message_ptr.null?
+        nil
+      else
+        # Create struct wrapper
+        native_message = Rdkafka::Bindings::Message.new(message_ptr)
+        # Raise error if needed
+        if native_message[:err] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
           raise Rdkafka::RdkafkaError.new(native_message[:err])
         end
         # Create a message to pass out
@@ -579,37 +764,182 @@ module Rdkafka
     # @note This method technically should be called `#poll` and the current `#poll` should be
     #   called `#consumer_poll` though we keep the current naming convention to make it backward
     #   compatible.
-    def events_poll(timeout_ms = 0)
+    def events_poll(timeout_ms = Defaults::CONSUMER_EVENTS_POLL_TIMEOUT_MS)
       @native_kafka.with_inner do |inner|
         Rdkafka::Bindings.rd_kafka_poll(inner, timeout_ms)
       end
     end
 
+    # Polls the main rdkafka queue without releasing the GVL (Global VM Lock).
+    #
+    # This is more efficient than regular events_poll for non-blocking poll(0) calls,
+    # particularly useful in fiber scheduler contexts where GVL release/reacquire
+    # overhead is wasteful since we don't expect to wait.
+    #
+    # @param timeout_ms [Integer] poll timeout (default: 0 for non-blocking)
+    # @return [Integer] the number of events served
+    #
+    # @see #events_poll for more details on when to use this method
+    def events_poll_nb(timeout_ms = 0)
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_poll_nb(inner, timeout_ms)
+      end
+    end
+
+    # Poll for a batch of messages from the consumer queue in a single FFI call.
+    #
+    # This is more efficient than calling {#poll} in a loop because it crosses the FFI
+    # boundary only once to fetch up to `max_items` messages.
+    #
+    # The timeout controls how long to wait for the **first** message. Once any message
+    # is available, librdkafka fills the buffer with whatever is immediately ready and
+    # returns without further waiting.
+    #
+    # @param timeout_ms [Integer] Timeout waiting for the first message (-1 for infinite)
+    # @param max_items [Integer] Maximum number of messages to return per call
+    # @return [Array<Message>] Array of messages (empty if none available within timeout)
+    # @raise [RdkafkaError] When a consumed message contains an error
+    # @raise [ClosedConsumerError] When called on a closed consumer
+    def poll_batch(timeout_ms, max_items: 100)
+      closed_consumer_check(__method__)
+
+      buffer = batch_buffer(max_items)
+      messages = []
+
+      count = @native_kafka.with_inner do |_inner|
+        Rdkafka::Bindings.rd_kafka_consume_batch_queue(
+          consumer_queue,
+          timeout_ms,
+          buffer,
+          max_items
+        )
+      end
+
+      return messages if count <= 0
+
+      i = 0
+      begin
+        while i < count
+          ptr = buffer.get_pointer(i * FFI::Pointer.size)
+
+          if ptr.null?
+            i += 1
+            next
+          end
+
+          native_message = Rdkafka::Bindings::Message.new(ptr)
+
+          if native_message[:err] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+            raise Rdkafka::RdkafkaError.new(native_message[:err])
+          end
+
+          messages << Rdkafka::Consumer::Message.new(native_message)
+          Rdkafka::Bindings.rd_kafka_message_destroy(ptr)
+          i += 1
+        end
+      ensure
+        while i < count
+          ptr = buffer.get_pointer(i * FFI::Pointer.size)
+          Rdkafka::Bindings.rd_kafka_message_destroy(ptr) unless ptr.null?
+          i += 1
+        end
+      end
+
+      messages
+    end
+
+    # Poll for a batch of messages without releasing the GVL (Global VM Lock).
+    #
+    # This is more efficient than {#poll_batch} for non-blocking poll(0) calls,
+    # particularly useful in fiber scheduler contexts where GVL release/reacquire
+    # overhead is wasteful since we don't expect to wait.
+    #
+    # @note Since the GVL is not released, a non-zero timeout_ms will block all Ruby
+    #   threads/fibers for the duration. Use {#poll_batch} if you need a blocking wait.
+    #
+    # @param timeout_ms [Integer] Timeout waiting for the first message (default: 0 for non-blocking)
+    # @param max_items [Integer] Maximum number of messages to return per call
+    # @return [Array<Message>] Array of messages (empty if none available within timeout)
+    # @raise [RdkafkaError] When a consumed message contains an error
+    # @raise [ClosedConsumerError] When called on a closed consumer
+    def poll_batch_nb(timeout_ms = 0, max_items: 100)
+      closed_consumer_check(__method__)
+
+      buffer = batch_buffer(max_items)
+      messages = []
+
+      count = @native_kafka.with_inner do |_inner|
+        Rdkafka::Bindings.rd_kafka_consume_batch_queue_nb(
+          consumer_queue,
+          timeout_ms,
+          buffer,
+          max_items
+        )
+      end
+
+      return messages if count <= 0
+
+      i = 0
+      begin
+        while i < count
+          ptr = buffer.get_pointer(i * FFI::Pointer.size)
+
+          if ptr.null?
+            i += 1
+            next
+          end
+
+          native_message = Rdkafka::Bindings::Message.new(ptr)
+
+          if native_message[:err] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+            raise Rdkafka::RdkafkaError.new(native_message[:err])
+          end
+
+          messages << Rdkafka::Consumer::Message.new(native_message)
+          Rdkafka::Bindings.rd_kafka_message_destroy(ptr)
+          i += 1
+        end
+      ensure
+        while i < count
+          ptr = buffer.get_pointer(i * FFI::Pointer.size)
+          Rdkafka::Bindings.rd_kafka_message_destroy(ptr) unless ptr.null?
+          i += 1
+        end
+      end
+
+      messages
+    end
+
     # Poll for new messages and yield for each received one. Iteration
     # will end when the consumer is closed.
     #
     # If `enable.partition.eof` is turned on in the config this will raise an error when an eof is
     # reached, so you probably want to disable that when using this method of iteration.
     #
+    # @param timeout_ms [Integer] Timeout for each poll iteration
     # @yieldparam message [Message] Received message
     # @return [nil]
     # @raise [RdkafkaError] When polling fails
-    def each
+    def each(timeout_ms: Defaults::CONSUMER_POLL_TIMEOUT_MS)
       loop do
-        message = poll(250)
+        message = poll(timeout_ms)
         if message
           yield(message)
+        elsif closed?
+          break
         else
-          if closed?
-            break
-          else
-            next
-          end
+          next
         end
       end
     end
 
-    # Deprecated. Please read the error message for more details.
+    # @deprecated This method has been removed due to data consistency concerns
+    # @param max_items [Integer] unused
+    # @param bytes_threshold [Numeric] unused
+    # @param timeout_ms [Integer] unused
+    # @param yield_on_error [Boolean] unused
+    # @param block [Proc] unused block
+    # @raise [NotImplementedError] Always raises as this method is no longer supported
     def each_batch(max_items: 100, bytes_threshold: Float::INFINITY, timeout_ms: 250, yield_on_error: false, &block)
       raise NotImplementedError, <<~ERROR
         `each_batch` has been removed due to data consistency concerns.
@@ -646,8 +976,30 @@ module Rdkafka
 
     private
 
+    # Checks if the consumer is closed and raises an error if so
+    # @param method [Symbol] name of the calling method for error context
+    # @raise [ClosedConsumerError] when the consumer is closed
     def closed_consumer_check(method)
       raise Rdkafka::ClosedConsumerError.new(method) if closed?
     end
+
+    # Returns the consumer queue pointer, lazily initialized
+    # @return [FFI::Pointer] consumer queue handle
+    def consumer_queue
+      @consumer_queue ||= @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_queue_get_consumer(inner)
+      end
+    end
+
+    # Returns a reusable FFI buffer for batch polling, growing if needed
+    # @param max_items [Integer] minimum buffer capacity
+    # @return [FFI::MemoryPointer] pointer buffer
+    def batch_buffer(max_items)
+      if @batch_buffer.nil? || @batch_buffer_size < max_items
+        @batch_buffer = FFI::MemoryPointer.new(:pointer, max_items)
+        @batch_buffer_size = max_items
+      end
+      @batch_buffer
+    end
   end
 end