karafka-rdkafka 0.24.0.rc2-aarch64-linux-gnu → 0.24.0.rc4-aarch64-linux-gnu

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d5a0a12c81986370bd3e74f1d50e072a1efaec78f87b9920ffe0c7a9633192c5
-  data.tar.gz: 3c1ff95662b2ba5bb6912ebeda520209f7c6b818762b3fb5f243a4751f0b2b10
+  metadata.gz: 67aea1d29ab7c9accce635b701dc80e09e2a07ee8208554c5a8be054aa85319d
+  data.tar.gz: ea2677a630149511b68987a5dd18f7908def22b7325540ae8e5b238d32e94ca4
 SHA512:
-  metadata.gz: c72c872626ef1d51b63f585a65a472d7d7b94fb050a610be91f409b686c010f64167d9bca633edbf2862ebb40c413156f9ae422045e103e2537e85e8bbb2cafd
-  data.tar.gz: f45068691155efba1f50b5f7b0794f2d23a2ae4e7f8db9651af2f5bed98e18c053a8c79e21ccbf19d6dfa434689740718e4e9e3b187b84a0d269fe17e822411a
+  metadata.gz: 72e001033ae77d321f97b1aa4d9bc15fa01bf99c1553b36f477f7d0c189fd8d89dd02866f9ce8989be1e3f7a4dca554af84543e5cbdc8d69cd2992220814defd
+  data.tar.gz: 690df2547a6bdafdd3716490181c78e6a1ff2b61f2008a62e0c9aa397560b7839d5740153ac850d23213e6c6d264ddbf75f1a6c24d8dfc059e1d79f6e6529a8b

data/CHANGELOG.md

@@ -15,6 +15,9 @@
 - [Enhancement] Use native ARM64 runners instead of QEMU emulation for Alpine musl aarch64 builds, improving build performance and reliability (from upstream).
 - [Enhancement] Enable parallel compilation (`make -j$(nproc)`) for ARM64 Alpine musl builds (from upstream).
 - [Enhancement] Bump librdkafka to 2.13.0.
+- [Enhancement] Add non-blocking poll methods (`poll_nb`, `events_poll_nb`) that skip GVL release for efficient fiber scheduler integration when using `poll(0)` (from upstream).
+- [Enhancement] Add `events_poll_nb_each` method on `Producer`, `Consumer`, and `Admin` for polling events in a single GVL/mutex session. Yields count after each iteration, caller returns `:stop` to break (from upstream).
+- [Enhancement] Add `poll_nb_each` method on `Consumer` for non-blocking message polling with proper resource cleanup, yielding each message and supporting early termination via `:stop` return value (from upstream).
 - [Fix] Fix Kerberos build on Alpine 3.23+ (GCC 15/C23) by forcing C17 semantics to maintain compatibility with old-style K&R declarations in MIT Kerberos and Cyrus SASL dependencies.
 
 ## 0.23.1 (2025-11-14)

data/lib/rdkafka/admin.rb

@@ -91,6 +91,45 @@ module Rdkafka
       @native_kafka.enable_background_queue_io_events(fd, payload)
     end
 
+    # Polls for events in a non-blocking loop, yielding the count after each iteration.
+    #
+    # This method processes events (stats, errors, etc.) in a single GVL/mutex session,
+    # which is more efficient than repeated individual polls. It uses non-blocking polls
+    # internally (no GVL release between polls).
+    #
+    # Yields the count of events processed after each poll iteration, allowing the caller
+    # to implement timeout or other termination logic by returning `:stop`.
+    #
+    # @yield [count] Called after each poll iteration
+    # @yieldparam count [Integer] Number of events processed in this iteration
+    # @yieldreturn [Symbol, Object] Return `:stop` to break the loop, any other value continues
+    # @return [nil]
+    # @raise [Rdkafka::ClosedAdminError] if called on a closed admin client
+    #
+    # @note This method holds the inner lock until the queue is empty or `:stop` is returned.
+    #   Other admin operations will wait until this method returns.
+    # @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
+    #
+    # @example Drain all pending events
+    #   admin.events_poll_nb_each { |_count| }
+    #
+    # @example With timeout control
+    #   deadline = monotonic_now + timeout_ms
+    #   admin.events_poll_nb_each do |_count|
+    #     :stop if monotonic_now >= deadline
+    #   end
+    def events_poll_nb_each
+      closed_admin_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        loop do
+          count = Rdkafka::Bindings.rd_kafka_poll_nb(inner, 0)
+          break if count.zero?
+          break if yield(count) == :stop
+        end
+      end
+    end
+
     # @return [Proc] finalizer proc for closing the admin
     # @private
     def finalizer

data/lib/rdkafka/bindings.rb

@@ -96,6 +96,12 @@ module Rdkafka
     attach_function :rd_kafka_poll, [:pointer, :int], :int, blocking: true
     attach_function :rd_kafka_outq_len, [:pointer], :int, blocking: true
 
+    # Non-blocking poll variants (do not release GVL)
+    # These are more efficient for poll(0) calls in fiber schedulers where GVL
+    # release/reacquire overhead is wasteful since we don't expect to wait.
+    # Uses the same underlying C function but with blocking: false to skip GVL release.
+    attach_function :rd_kafka_poll_nb, :rd_kafka_poll, [:pointer, :int], :int, blocking: false
+
     # Metadata
 
     attach_function :rd_kafka_name, [:pointer], :string
@@ -398,6 +404,9 @@ module Rdkafka
     attach_function :rd_kafka_commit, [:pointer, :pointer, :bool], :int, blocking: true
     attach_function :rd_kafka_poll_set_consumer, [:pointer], :void, blocking: true
     attach_function :rd_kafka_consumer_poll, [:pointer, :int], :pointer, blocking: true
+    # Non-blocking consumer poll variant (does not release GVL)
+    # More efficient for poll(0) calls in fiber schedulers.
+    attach_function :rd_kafka_consumer_poll_nb, :rd_kafka_consumer_poll, [:pointer, :int], :pointer, blocking: false
     attach_function :rd_kafka_consumer_close, [:pointer], :void, blocking: true
     attach_function :rd_kafka_offsets_store, [:pointer, :pointer], :int, blocking: true
     attach_function :rd_kafka_pause_partitions, [:pointer, :pointer], :int, blocking: true

data/lib/rdkafka/consumer.rb

@@ -75,6 +75,103 @@ module Rdkafka
       @native_kafka.enable_background_queue_io_events(fd, payload)
     end
 
+    # Polls for events in a non-blocking loop, yielding the count after each iteration.
+    #
+    # This method processes events (stats, errors, etc.) in a single GVL/mutex session,
+    # which is more efficient than repeated individual polls. It uses non-blocking polls
+    # internally (no GVL release between polls).
+    #
+    # Yields the count of events processed after each poll iteration, allowing the caller
+    # to implement timeout or other termination logic by returning `:stop`.
+    #
+    # @yield [count] Called after each poll iteration
+    # @yieldparam count [Integer] Number of events processed in this iteration
+    # @yieldreturn [Symbol, Object] Return `:stop` to break the loop, any other value continues
+    # @return [nil]
+    # @raise [Rdkafka::ClosedConsumerError] if called on a closed consumer
+    #
+    # @note This method holds the inner lock until the queue is empty or `:stop` is returned.
+    #   Other consumer operations will wait until this method returns.
+    # @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
+    # @note Do NOT use this if `consumer_poll_set` was set to `true`
+    #
+    # @example Drain all pending events
+    #   consumer.events_poll_nb_each { |_count| }
+    #
+    # @example With timeout control
+    #   deadline = monotonic_now + timeout_ms
+    #   consumer.events_poll_nb_each do |_count|
+    #     :stop if monotonic_now >= deadline
+    #   end
+    def events_poll_nb_each
+      closed_consumer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        loop do
+          count = Rdkafka::Bindings.rd_kafka_poll_nb(inner, 0)
+          break if count.zero?
+          break if yield(count) == :stop
+        end
+      end
+    end
+
+    # Polls for messages in a non-blocking loop, yielding each message to the caller.
+    #
+    # This method processes messages in a single GVL/mutex session until the queue is empty
+    # or the caller returns `:stop`. It handles the message pointer lifecycle internally,
+    # ensuring proper cleanup via `rd_kafka_message_destroy`.
+    #
+    # @yield [message] Called for each message received
+    # @yieldparam message [Consumer::Message] The received message
+    # @yieldreturn [Symbol, Object] Return `:stop` to break the loop, any other value continues
+    # @return [nil]
+    # @raise [Rdkafka::ClosedConsumerError] if called on a closed consumer
+    # @raise [Rdkafka::RdkafkaError] if a Kafka error occurs while polling
+    #
+    # @note This method uses `rd_kafka_consumer_poll` to fetch messages, unlike
+    #   `events_poll_nb_each` which uses `rd_kafka_poll` for event callbacks (delivery reports,
+    #   statistics, etc.). For consumers, use this method to receive messages and
+    #   `events_poll_nb_each` for processing background events.
+    # @note This method holds the inner lock for the duration. Other consumer operations
+    #   will wait until this method returns.
+    # @note Timeout/max_messages logic should be implemented by the caller
+    #
+    # @example Process messages until queue is empty
+    #   consumer.poll_nb_each do |message|
+    #     process(message)
+    #   end
+    #
+    # @example Process with early termination
+    #   count = 0
+    #   consumer.poll_nb_each do |message|
+    #     process(message)
+    #     count += 1
+    #     :stop if count >= 10
+    #   end
+    def poll_nb_each
+      closed_consumer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        loop do
+          message_ptr = Rdkafka::Bindings.rd_kafka_consumer_poll_nb(inner, 0)
+          break if message_ptr.null?
+
+          begin
+            native_message = Rdkafka::Bindings::Message.new(message_ptr)
+
+            if native_message[:err] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+              raise Rdkafka::RdkafkaError.new(native_message[:err])
+            end
+
+            result = yield Consumer::Message.new(native_message)
+            break if result == :stop
+          ensure
+            Rdkafka::Bindings.rd_kafka_message_destroy(message_ptr)
+          end
+        end
+      end
+    end
+
     # @return [Proc] finalizer proc for closing the consumer
     # @private
     def finalizer
@@ -606,6 +703,47 @@ module Rdkafka
       end
     end
 
+    # Poll for the next message without releasing the GVL (Global VM Lock).
+    #
+    # This is more efficient than regular polling for non-blocking poll(0) calls,
+    # particularly useful in fiber scheduler contexts where GVL release/reacquire
+    # overhead is wasteful since we don't expect to wait.
+    #
+    # @param timeout_ms [Integer] Timeout of this poll (default: 0 for non-blocking)
+    # @return [Message, nil] A message or nil if there was no new message within the timeout
+    # @raise [RdkafkaError] When polling fails
+    #
+    # @example Using with fiber scheduler
+    #   # After receiving IO notification that messages are available
+    #   while msg = consumer.poll_nb
+    #     process(msg)
+    #   end
+    def poll_nb(timeout_ms = 0)
+      closed_consumer_check(__method__)
+
+      message_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_consumer_poll_nb(inner, timeout_ms)
+      end
+
+      if message_ptr.null?
+        nil
+      else
+        # Create struct wrapper
+        native_message = Rdkafka::Bindings::Message.new(message_ptr)
+        # Raise error if needed
+        if native_message[:err] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+          raise Rdkafka::RdkafkaError.new(native_message[:err])
+        end
+        # Create a message to pass out
+        Rdkafka::Consumer::Message.new(native_message)
+      end
+    ensure
+      # Clean up rdkafka message if there is one
+      if message_ptr && !message_ptr.null?
+        Rdkafka::Bindings.rd_kafka_message_destroy(message_ptr)
+      end
+    end
+
     # Polls the main rdkafka queue (not the consumer one). Do **NOT** use it if `consumer_poll_set`
     #   was set to `true`.
     #
@@ -632,6 +770,22 @@ module Rdkafka
       end
     end
 
+    # Polls the main rdkafka queue without releasing the GVL (Global VM Lock).
+    #
+    # This is more efficient than regular events_poll for non-blocking poll(0) calls,
+    # particularly useful in fiber scheduler contexts where GVL release/reacquire
+    # overhead is wasteful since we don't expect to wait.
+    #
+    # @param timeout_ms [Integer] poll timeout (default: 0 for non-blocking)
+    # @return [Integer] the number of events served
+    #
+    # @see #events_poll for more details on when to use this method
+    def events_poll_nb(timeout_ms = 0)
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_poll_nb(inner, timeout_ms)
+      end
+    end
+
     # Poll for new messages and yield for each received one. Iteration
     # will end when the consumer is closed.
     #

data/lib/rdkafka/helpers/time.rb

@@ -9,6 +9,11 @@ module Rdkafka
       def monotonic_now
         ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       end
+
+      # @return [Integer] current monotonic time in milliseconds
+      def monotonic_now_ms
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :millisecond)
+      end
     end
   end
 end

data/lib/rdkafka/producer/partitions_count_cache.rb

@@ -147,17 +147,17 @@ module Rdkafka
 
           if current_info.nil?
             # Create new entry
-            @counts[topic] = [monotonic_now, new_count]
+            @counts[topic] = [monotonic_now_ms, new_count]
           else
             current_count = current_info[1]
 
             if new_count > current_count
               # Update to higher count value
-              current_info[0] = monotonic_now
+              current_info[0] = monotonic_now_ms
               current_info[1] = new_count
             else
               # Same or lower count, update timestamp only
-              current_info[0] = monotonic_now
+              current_info[0] = monotonic_now_ms
             end
           end
         end
@@ -211,15 +211,15 @@ module Rdkafka
         return unless current_info
 
         # Update the timestamp in-place
-        current_info[0] = monotonic_now
+        current_info[0] = monotonic_now_ms
       end
 
       # Check if a timestamp has expired based on the TTL
       #
-      # @param timestamp [Float] Monotonic timestamp to check
+      # @param timestamp [Integer] Monotonic timestamp in milliseconds to check
       # @return [Boolean] true if expired, false otherwise
       def expired?(timestamp)
-        (monotonic_now - timestamp) * 1_000 > @ttl_ms
+        monotonic_now_ms - timestamp > @ttl_ms
       end
     end
   end

data/lib/rdkafka/producer.rb

@@ -338,6 +338,45 @@ module Rdkafka
 
     alias_method :queue_length, :queue_size
 
+    # Polls for events in a non-blocking loop, yielding the count after each iteration.
+    #
+    # This method processes delivery callbacks in a single GVL/mutex session, which is more
+    # efficient than repeated individual polls. It uses non-blocking polls internally
+    # (no GVL release between polls).
+    #
+    # Yields the count of events processed after each poll iteration, allowing the caller
+    # to implement timeout or other termination logic by returning `:stop`.
+    #
+    # @yield [count] Called after each poll iteration
+    # @yieldparam count [Integer] Number of events processed in this iteration
+    # @yieldreturn [Symbol, Object] Return `:stop` to break the loop, any other value continues
+    # @return [nil]
+    # @raise [Rdkafka::ClosedProducerError] if called on a closed producer
+    #
+    # @note This method holds the inner lock until the queue is empty or `:stop` is returned.
+    #   Other producer operations (produce, close, etc.) will wait until this method returns.
+    # @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
+    #
+    # @example Drain all pending callbacks
+    #   producer.events_poll_nb_each { |_count| }
+    #
+    # @example With timeout control
+    #   deadline = monotonic_now + timeout_ms
+    #   producer.events_poll_nb_each do |_count|
+    #     :stop if monotonic_now >= deadline
+    #   end
+    def events_poll_nb_each
+      closed_producer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        loop do
+          count = Rdkafka::Bindings.rd_kafka_poll_nb(inner, 0)
+          break if count.zero?
+          break if yield(count) == :stop
+        end
+      end
+    end
+
     # Partition count for a given topic.
     #
     # @param topic [String] The topic name.

data/lib/rdkafka/version.rb

@@ -2,7 +2,7 @@
 
 module Rdkafka
   # Current rdkafka-ruby gem version
-  VERSION = "0.24.0.rc2"
+  VERSION = "0.24.0.rc4"
   # Target librdkafka version to be used
   LIBRDKAFKA_VERSION = "2.13.0"
   # SHA256 hash of the librdkafka source tarball for verification

data/renovate.json

@@ -5,6 +5,7 @@
   ],
   "minimumReleaseAge": "7 days",
   "includePaths": [
+    ".ruby-version",
     "Gemfile",
     "Gemfile.lint",
     "package.json",

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.24.0.rc2
+  version: 0.24.0.rc4
 platform: aarch64-linux-gnu
 authors:
 - Thijs Cadier