karafka-rdkafka 0.24.0.rc2-aarch64-linux-gnu → 0.24.0.rc3-aarch64-linux-gnu

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d5a0a12c81986370bd3e74f1d50e072a1efaec78f87b9920ffe0c7a9633192c5
-  data.tar.gz: 3c1ff95662b2ba5bb6912ebeda520209f7c6b818762b3fb5f243a4751f0b2b10
+  metadata.gz: 0fc37d005a8920103966e128744f2afefb21867046b43777d31cb5c40bc995f1
+  data.tar.gz: 44a57e87fb413e25815419909cdb2de0f138e4c6cd83e10ef3ee0495bb324f4c
 SHA512:
-  metadata.gz: c72c872626ef1d51b63f585a65a472d7d7b94fb050a610be91f409b686c010f64167d9bca633edbf2862ebb40c413156f9ae422045e103e2537e85e8bbb2cafd
-  data.tar.gz: f45068691155efba1f50b5f7b0794f2d23a2ae4e7f8db9651af2f5bed98e18c053a8c79e21ccbf19d6dfa434689740718e4e9e3b187b84a0d269fe17e822411a
+  metadata.gz: 52b918e97d0db2b47bcdc8adbe73c24df0a82fb0cf3854cff644735de31cc577b509c8fd10d19a29087e6a419feacf930cb73f680f081deeb612933e3e402d46
+  data.tar.gz: bc22e5d641c2e3bdac08844e6899e7ba750ddb4b3c1a7345175b2678f24f9a31fe088ef04799968c091520311398e616ce2377856ad90887d628bbdf3c825e9b

data/CHANGELOG.md

@@ -15,6 +15,7 @@
 - [Enhancement] Use native ARM64 runners instead of QEMU emulation for Alpine musl aarch64 builds, improving build performance and reliability (from upstream).
 - [Enhancement] Enable parallel compilation (`make -j$(nproc)`) for ARM64 Alpine musl builds (from upstream).
 - [Enhancement] Bump librdkafka to 2.13.0.
+- [Enhancement] Add non-blocking poll methods (`poll_nb`, `events_poll_nb`) that skip GVL release for efficient fiber scheduler integration when using `poll(0)` (from upstream).
 - [Fix] Fix Kerberos build on Alpine 3.23+ (GCC 15/C23) by forcing C17 semantics to maintain compatibility with old-style K&R declarations in MIT Kerberos and Cyrus SASL dependencies.
 
 ## 0.23.1 (2025-11-14)

data/lib/rdkafka/bindings.rb

@@ -96,6 +96,12 @@ module Rdkafka
     attach_function :rd_kafka_poll, [:pointer, :int], :int, blocking: true
     attach_function :rd_kafka_outq_len, [:pointer], :int, blocking: true
 
+    # Non-blocking poll variants (do not release GVL)
+    # These are more efficient for poll(0) calls in fiber schedulers where GVL
+    # release/reacquire overhead is wasteful since we don't expect to wait.
+    # Uses the same underlying C function but with blocking: false to skip GVL release.
+    attach_function :rd_kafka_poll_nb, :rd_kafka_poll, [:pointer, :int], :int, blocking: false
+
     # Metadata
 
     attach_function :rd_kafka_name, [:pointer], :string
@@ -398,6 +404,9 @@ module Rdkafka
     attach_function :rd_kafka_commit, [:pointer, :pointer, :bool], :int, blocking: true
     attach_function :rd_kafka_poll_set_consumer, [:pointer], :void, blocking: true
     attach_function :rd_kafka_consumer_poll, [:pointer, :int], :pointer, blocking: true
+    # Non-blocking consumer poll variant (does not release GVL)
+    # More efficient for poll(0) calls in fiber schedulers.
+    attach_function :rd_kafka_consumer_poll_nb, :rd_kafka_consumer_poll, [:pointer, :int], :pointer, blocking: false
     attach_function :rd_kafka_consumer_close, [:pointer], :void, blocking: true
     attach_function :rd_kafka_offsets_store, [:pointer, :pointer], :int, blocking: true
     attach_function :rd_kafka_pause_partitions, [:pointer, :pointer], :int, blocking: true

data/lib/rdkafka/consumer.rb

@@ -606,6 +606,47 @@ module Rdkafka
       end
     end
 
+    # Poll for the next message without releasing the GVL (Global VM Lock).
+    #
+    # This is more efficient than regular polling for non-blocking poll(0) calls,
+    # particularly useful in fiber scheduler contexts where GVL release/reacquire
+    # overhead is wasteful since we don't expect to wait.
+    #
+    # @param timeout_ms [Integer] Timeout of this poll (default: 0 for non-blocking)
+    # @return [Message, nil] A message or nil if there was no new message within the timeout
+    # @raise [RdkafkaError] When polling fails
+    #
+    # @example Using with fiber scheduler
+    #   # After receiving IO notification that messages are available
+    #   while msg = consumer.poll_nb
+    #     process(msg)
+    #   end
+    def poll_nb(timeout_ms = 0)
+      closed_consumer_check(__method__)
+
+      message_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_consumer_poll_nb(inner, timeout_ms)
+      end
+
+      if message_ptr.null?
+        nil
+      else
+        # Create struct wrapper
+        native_message = Rdkafka::Bindings::Message.new(message_ptr)
+        # Raise error if needed
+        if native_message[:err] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+          raise Rdkafka::RdkafkaError.new(native_message[:err])
+        end
+        # Create a message to pass out
+        Rdkafka::Consumer::Message.new(native_message)
+      end
+    ensure
+      # Clean up rdkafka message if there is one
+      if message_ptr && !message_ptr.null?
+        Rdkafka::Bindings.rd_kafka_message_destroy(message_ptr)
+      end
+    end
+
     # Polls the main rdkafka queue (not the consumer one). Do **NOT** use it if `consumer_poll_set`
     #   was set to `true`.
     #
@@ -632,6 +673,22 @@ module Rdkafka
       end
     end
 
+    # Polls the main rdkafka queue without releasing the GVL (Global VM Lock).
+    #
+    # This is more efficient than regular events_poll for non-blocking poll(0) calls,
+    # particularly useful in fiber scheduler contexts where GVL release/reacquire
+    # overhead is wasteful since we don't expect to wait.
+    #
+    # @param timeout_ms [Integer] poll timeout (default: 0 for non-blocking)
+    # @return [Integer] the number of events served
+    #
+    # @see #events_poll for more details on when to use this method
+    def events_poll_nb(timeout_ms = 0)
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_poll_nb(inner, timeout_ms)
+      end
+    end
+
     # Poll for new messages and yield for each received one. Iteration
     # will end when the consumer is closed.
     #

data/lib/rdkafka/helpers/time.rb

@@ -9,6 +9,11 @@ module Rdkafka
       def monotonic_now
         ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       end
+
+      # @return [Integer] current monotonic time in milliseconds
+      def monotonic_now_ms
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :millisecond)
+      end
     end
   end
 end

data/lib/rdkafka/producer/partitions_count_cache.rb

@@ -147,17 +147,17 @@ module Rdkafka
 
           if current_info.nil?
             # Create new entry
-            @counts[topic] = [monotonic_now, new_count]
+            @counts[topic] = [monotonic_now_ms, new_count]
           else
             current_count = current_info[1]
 
             if new_count > current_count
               # Update to higher count value
-              current_info[0] = monotonic_now
+              current_info[0] = monotonic_now_ms
               current_info[1] = new_count
             else
               # Same or lower count, update timestamp only
-              current_info[0] = monotonic_now
+              current_info[0] = monotonic_now_ms
             end
           end
         end
@@ -211,15 +211,15 @@ module Rdkafka
         return unless current_info
 
         # Update the timestamp in-place
-        current_info[0] = monotonic_now
+        current_info[0] = monotonic_now_ms
       end
 
       # Check if a timestamp has expired based on the TTL
       #
-      # @param timestamp [Float] Monotonic timestamp to check
+      # @param timestamp [Integer] Monotonic timestamp in milliseconds to check
       # @return [Boolean] true if expired, false otherwise
       def expired?(timestamp)
-        (monotonic_now - timestamp) * 1_000 > @ttl_ms
+        monotonic_now_ms - timestamp > @ttl_ms
       end
     end
   end

data/lib/rdkafka/producer.rb

@@ -338,6 +338,45 @@ module Rdkafka
 
     alias_method :queue_length, :queue_size
 
+    # Drains the producer's event queue by continuously polling until empty or time limit reached.
+    #
+    # This method is useful when you need to ensure delivery callbacks are processed within a
+    # bounded time, particularly when polling multiple producers from a single thread where
+    # fair scheduling is required to prevent starvation.
+    #
+    # Uses non-blocking polls internally (no GVL release) for efficiency. The method holds
+    # a single `with_inner` lock for the duration, minimizing per-poll overhead when processing
+    # many events.
+    #
+    # @param timeout_ms [Integer] maximum time to spend draining in milliseconds (default: 100)
+    # @return [Boolean] true if no more events to process, false if stopped due to time limit
+    # @raise [Rdkafka::ClosedProducerError] if called on a closed producer
+    #
+    # @note This method holds the inner lock for up to `timeout_ms`. Other producer operations
+    #   (produce, close, etc.) will wait until this method returns.
+    # @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
+    #
+    # @example Basic usage - drain for up to 100ms
+    #   fully_drained = producer.poll_drain_nb
+    #
+    # @example Round-robin polling multiple producers fairly
+    #   producers.each do |producer|
+    #     fully_drained = producer.poll_drain_nb(10)
+    #     # If false, this producer has more pending events
+    #   end
+    def poll_drain_nb(timeout_ms = 100)
+      closed_producer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        deadline = monotonic_now_ms + timeout_ms
+
+        loop do
+          break true if Rdkafka::Bindings.rd_kafka_poll_nb(inner, 0).zero?
+          break false if monotonic_now_ms >= deadline
+        end
+      end
+    end
+
     # Partition count for a given topic.
     #
     # @param topic [String] The topic name.

data/lib/rdkafka/version.rb

@@ -2,7 +2,7 @@
 
 module Rdkafka
   # Current rdkafka-ruby gem version
-  VERSION = "0.24.0.rc2"
+  VERSION = "0.24.0.rc3"
   # Target librdkafka version to be used
   LIBRDKAFKA_VERSION = "2.13.0"
   # SHA256 hash of the librdkafka source tarball for verification

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.24.0.rc2
+  version: 0.24.0.rc3
 platform: aarch64-linux-gnu
 authors:
 - Thijs Cadier