karafka-rdkafka 0.24.0.rc3-aarch64-linux-gnu → 0.24.0.rc4-aarch64-linux-gnu

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0fc37d005a8920103966e128744f2afefb21867046b43777d31cb5c40bc995f1
-  data.tar.gz: 44a57e87fb413e25815419909cdb2de0f138e4c6cd83e10ef3ee0495bb324f4c
+  metadata.gz: 67aea1d29ab7c9accce635b701dc80e09e2a07ee8208554c5a8be054aa85319d
+  data.tar.gz: ea2677a630149511b68987a5dd18f7908def22b7325540ae8e5b238d32e94ca4
 SHA512:
-  metadata.gz: 52b918e97d0db2b47bcdc8adbe73c24df0a82fb0cf3854cff644735de31cc577b509c8fd10d19a29087e6a419feacf930cb73f680f081deeb612933e3e402d46
-  data.tar.gz: bc22e5d641c2e3bdac08844e6899e7ba750ddb4b3c1a7345175b2678f24f9a31fe088ef04799968c091520311398e616ce2377856ad90887d628bbdf3c825e9b
+  metadata.gz: 72e001033ae77d321f97b1aa4d9bc15fa01bf99c1553b36f477f7d0c189fd8d89dd02866f9ce8989be1e3f7a4dca554af84543e5cbdc8d69cd2992220814defd
+  data.tar.gz: 690df2547a6bdafdd3716490181c78e6a1ff2b61f2008a62e0c9aa397560b7839d5740153ac850d23213e6c6d264ddbf75f1a6c24d8dfc059e1d79f6e6529a8b

data/CHANGELOG.md

@@ -16,6 +16,8 @@
 - [Enhancement] Enable parallel compilation (`make -j$(nproc)`) for ARM64 Alpine musl builds (from upstream).
 - [Enhancement] Bump librdkafka to 2.13.0.
 - [Enhancement] Add non-blocking poll methods (`poll_nb`, `events_poll_nb`) that skip GVL release for efficient fiber scheduler integration when using `poll(0)` (from upstream).
+- [Enhancement] Add `events_poll_nb_each` method on `Producer`, `Consumer`, and `Admin` for polling events in a single GVL/mutex session. Yields count after each iteration, caller returns `:stop` to break (from upstream).
+- [Enhancement] Add `poll_nb_each` method on `Consumer` for non-blocking message polling with proper resource cleanup, yielding each message and supporting early termination via `:stop` return value (from upstream).
 - [Fix] Fix Kerberos build on Alpine 3.23+ (GCC 15/C23) by forcing C17 semantics to maintain compatibility with old-style K&R declarations in MIT Kerberos and Cyrus SASL dependencies.
 
 ## 0.23.1 (2025-11-14)

data/lib/rdkafka/admin.rb

@@ -91,6 +91,45 @@ module Rdkafka
       @native_kafka.enable_background_queue_io_events(fd, payload)
     end
 
+    # Polls for events in a non-blocking loop, yielding the count after each iteration.
+    #
+    # This method processes events (stats, errors, etc.) in a single GVL/mutex session,
+    # which is more efficient than repeated individual polls. It uses non-blocking polls
+    # internally (no GVL release between polls).
+    #
+    # Yields the count of events processed after each poll iteration, allowing the caller
+    # to implement timeout or other termination logic by returning `:stop`.
+    #
+    # @yield [count] Called after each poll iteration
+    # @yieldparam count [Integer] Number of events processed in this iteration
+    # @yieldreturn [Symbol, Object] Return `:stop` to break the loop, any other value continues
+    # @return [nil]
+    # @raise [Rdkafka::ClosedAdminError] if called on a closed admin client
+    #
+    # @note This method holds the inner lock until the queue is empty or `:stop` is returned.
+    #   Other admin operations will wait until this method returns.
+    # @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
+    #
+    # @example Drain all pending events
+    #   admin.events_poll_nb_each { |_count| }
+    #
+    # @example With timeout control
+    #   deadline = monotonic_now + timeout_ms
+    #   admin.events_poll_nb_each do |_count|
+    #     :stop if monotonic_now >= deadline
+    #   end
+    def events_poll_nb_each
+      closed_admin_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        loop do
+          count = Rdkafka::Bindings.rd_kafka_poll_nb(inner, 0)
+          break if count.zero?
+          break if yield(count) == :stop
+        end
+      end
+    end
+
     # @return [Proc] finalizer proc for closing the admin
     # @private
     def finalizer

data/lib/rdkafka/consumer.rb

@@ -75,6 +75,103 @@ module Rdkafka
       @native_kafka.enable_background_queue_io_events(fd, payload)
     end
 
+    # Polls for events in a non-blocking loop, yielding the count after each iteration.
+    #
+    # This method processes events (stats, errors, etc.) in a single GVL/mutex session,
+    # which is more efficient than repeated individual polls. It uses non-blocking polls
+    # internally (no GVL release between polls).
+    #
+    # Yields the count of events processed after each poll iteration, allowing the caller
+    # to implement timeout or other termination logic by returning `:stop`.
+    #
+    # @yield [count] Called after each poll iteration
+    # @yieldparam count [Integer] Number of events processed in this iteration
+    # @yieldreturn [Symbol, Object] Return `:stop` to break the loop, any other value continues
+    # @return [nil]
+    # @raise [Rdkafka::ClosedConsumerError] if called on a closed consumer
+    #
+    # @note This method holds the inner lock until the queue is empty or `:stop` is returned.
+    #   Other consumer operations will wait until this method returns.
+    # @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
+    # @note Do NOT use this if `consumer_poll_set` was set to `true`
+    #
+    # @example Drain all pending events
+    #   consumer.events_poll_nb_each { |_count| }
+    #
+    # @example With timeout control
+    #   deadline = monotonic_now + timeout_ms
+    #   consumer.events_poll_nb_each do |_count|
+    #     :stop if monotonic_now >= deadline
+    #   end
+    def events_poll_nb_each
+      closed_consumer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        loop do
+          count = Rdkafka::Bindings.rd_kafka_poll_nb(inner, 0)
+          break if count.zero?
+          break if yield(count) == :stop
+        end
+      end
+    end
+
+    # Polls for messages in a non-blocking loop, yielding each message to the caller.
+    #
+    # This method processes messages in a single GVL/mutex session until the queue is empty
+    # or the caller returns `:stop`. It handles the message pointer lifecycle internally,
+    # ensuring proper cleanup via `rd_kafka_message_destroy`.
+    #
+    # @yield [message] Called for each message received
+    # @yieldparam message [Consumer::Message] The received message
+    # @yieldreturn [Symbol, Object] Return `:stop` to break the loop, any other value continues
+    # @return [nil]
+    # @raise [Rdkafka::ClosedConsumerError] if called on a closed consumer
+    # @raise [Rdkafka::RdkafkaError] if a Kafka error occurs while polling
+    #
+    # @note This method uses `rd_kafka_consumer_poll` to fetch messages, unlike
+    #   `events_poll_nb_each` which uses `rd_kafka_poll` for event callbacks (delivery reports,
+    #   statistics, etc.). For consumers, use this method to receive messages and
+    #   `events_poll_nb_each` for processing background events.
+    # @note This method holds the inner lock for the duration. Other consumer operations
+    #   will wait until this method returns.
+    # @note Timeout/max_messages logic should be implemented by the caller
+    #
+    # @example Process messages until queue is empty
+    #   consumer.poll_nb_each do |message|
+    #     process(message)
+    #   end
+    #
+    # @example Process with early termination
+    #   count = 0
+    #   consumer.poll_nb_each do |message|
+    #     process(message)
+    #     count += 1
+    #     :stop if count >= 10
+    #   end
+    def poll_nb_each
+      closed_consumer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        loop do
+          message_ptr = Rdkafka::Bindings.rd_kafka_consumer_poll_nb(inner, 0)
+          break if message_ptr.null?
+
+          begin
+            native_message = Rdkafka::Bindings::Message.new(message_ptr)
+
+            if native_message[:err] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+              raise Rdkafka::RdkafkaError.new(native_message[:err])
+            end
+
+            result = yield Consumer::Message.new(native_message)
+            break if result == :stop
+          ensure
+            Rdkafka::Bindings.rd_kafka_message_destroy(message_ptr)
+          end
+        end
+      end
+    end
+
     # @return [Proc] finalizer proc for closing the consumer
     # @private
     def finalizer

data/lib/rdkafka/producer.rb

@@ -338,41 +338,41 @@ module Rdkafka
 
     alias_method :queue_length, :queue_size
 
-    # Drains the producer's event queue by continuously polling until empty or time limit reached.
+    # Polls for events in a non-blocking loop, yielding the count after each iteration.
     #
-    # This method is useful when you need to ensure delivery callbacks are processed within a
-    # bounded time, particularly when polling multiple producers from a single thread where
-    # fair scheduling is required to prevent starvation.
+    # This method processes delivery callbacks in a single GVL/mutex session, which is more
+    # efficient than repeated individual polls. It uses non-blocking polls internally
+    # (no GVL release between polls).
     #
-    # Uses non-blocking polls internally (no GVL release) for efficiency. The method holds
-    # a single `with_inner` lock for the duration, minimizing per-poll overhead when processing
-    # many events.
+    # Yields the count of events processed after each poll iteration, allowing the caller
+    # to implement timeout or other termination logic by returning `:stop`.
     #
-    # @param timeout_ms [Integer] maximum time to spend draining in milliseconds (default: 100)
-    # @return [Boolean] true if no more events to process, false if stopped due to time limit
+    # @yield [count] Called after each poll iteration
+    # @yieldparam count [Integer] Number of events processed in this iteration
+    # @yieldreturn [Symbol, Object] Return `:stop` to break the loop, any other value continues
+    # @return [nil]
     # @raise [Rdkafka::ClosedProducerError] if called on a closed producer
     #
-    # @note This method holds the inner lock for up to `timeout_ms`. Other producer operations
-    #   (produce, close, etc.) will wait until this method returns.
+    # @note This method holds the inner lock until the queue is empty or `:stop` is returned.
+    #   Other producer operations (produce, close, etc.) will wait until this method returns.
     # @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
     #
-    # @example Basic usage - drain for up to 100ms
-    #   fully_drained = producer.poll_drain_nb
+    # @example Drain all pending callbacks
+    #   producer.events_poll_nb_each { |_count| }
     #
-    # @example Round-robin polling multiple producers fairly
-    #   producers.each do |producer|
-    #     fully_drained = producer.poll_drain_nb(10)
-    #     # If false, this producer has more pending events
+    # @example With timeout control
+    #   deadline = monotonic_now + timeout_ms
+    #   producer.events_poll_nb_each do |_count|
+    #     :stop if monotonic_now >= deadline
     #   end
-    def poll_drain_nb(timeout_ms = 100)
+    def events_poll_nb_each
       closed_producer_check(__method__)
 
       @native_kafka.with_inner do |inner|
-        deadline = monotonic_now_ms + timeout_ms
-
         loop do
-          break true if Rdkafka::Bindings.rd_kafka_poll_nb(inner, 0).zero?
-          break false if monotonic_now_ms >= deadline
+          count = Rdkafka::Bindings.rd_kafka_poll_nb(inner, 0)
+          break if count.zero?
+          break if yield(count) == :stop
         end
       end
     end

data/lib/rdkafka/version.rb

@@ -2,7 +2,7 @@
 
 module Rdkafka
   # Current rdkafka-ruby gem version
-  VERSION = "0.24.0.rc3"
+  VERSION = "0.24.0.rc4"
   # Target librdkafka version to be used
   LIBRDKAFKA_VERSION = "2.13.0"
   # SHA256 hash of the librdkafka source tarball for verification

data/renovate.json

@@ -5,6 +5,7 @@
   ],
   "minimumReleaseAge": "7 days",
   "includePaths": [
+    ".ruby-version",
     "Gemfile",
     "Gemfile.lint",
     "package.json",

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.24.0.rc3
+  version: 0.24.0.rc4
 platform: aarch64-linux-gnu
 authors:
 - Thijs Cadier