waterdrop 2.8.14 → 2.8.16

data/lib/waterdrop/producer/buffer.rb

@@ -13,7 +13,7 @@ module WaterDrop
         ensure_active!
 
         @monitor.instrument(
-          'message.buffered',
+          "message.buffered",
           producer_id: id,
           message: message,
           buffer: @messages
@@ -30,7 +30,7 @@ module WaterDrop
         ensure_active!
 
         @monitor.instrument(
-          'messages.buffered',
+          "messages.buffered",
           producer_id: id,
           messages: messages,
           buffer: @messages
@@ -45,18 +45,18 @@ module WaterDrop
       #   flushed
       def flush_async
         @monitor.instrument(
-          'buffer.flushed_async',
+          "buffer.flushed_async",
           producer_id: id,
           messages: @messages
         ) { flush(false) }
       end
 
       # Flushes the internal buffer to Kafka in a sync way
-      # @return [Array<Rdkafka::Producer::DeliveryReport>] delivery reports for messages that were
-      #   flushed
+      # @return [Array<Rdkafka::Producer::DeliveryHandle>] delivery handles for messages that were
+      #   flushed (handles are in final state, call `#create_result` to get delivery report)
       def flush_sync
         @monitor.instrument(
-          'buffer.flushed_sync',
+          "buffer.flushed_sync",
           producer_id: id,
           messages: @messages
         ) { flush(true) }
@@ -66,8 +66,8 @@ module WaterDrop
 
       # Method for triggering the buffer
       # @param sync [Boolean] should it flush in a sync way
-      # @return [Array<Rdkafka::Producer::DeliveryHandle, Rdkafka::Producer::DeliveryReport>]
-      #   delivery handles for async or delivery reports for sync
+      # @return [Array<Rdkafka::Producer::DeliveryHandle>] delivery handles (in final state for
+      #   sync, pending for async)
       # @raise [Errors::ProduceManyError] when there was a failure in flushing
       # @note We use this method underneath to provide a different instrumentation for sync and
       #   async flushing within the public API

data/lib/waterdrop/producer/idempotence.rb

@@ -10,7 +10,7 @@ module WaterDrop
         return true if transactional?
         return @idempotent unless @idempotent.nil?
 
-        @idempotent = config.kafka.to_h.fetch(:'enable.idempotence', false)
+        @idempotent = config.kafka.to_h.fetch(:"enable.idempotence", false)
       end
 
       # Checks if the given error should trigger an idempotent producer reload
@@ -61,7 +61,7 @@ module WaterDrop
           # Users can subscribe to this event and modify event[:caller].config.kafka to change
           # producer config
           @monitor.instrument(
-            'producer.reload',
+            "producer.reload",
             producer_id: id,
             error: error,
             attempt: attempt,
@@ -73,7 +73,7 @@ module WaterDrop
           @idempotent = nil
 
           @monitor.instrument(
-            'producer.reloaded',
+            "producer.reloaded",
             producer_id: id,
             attempt: attempt
           ) do

data/lib/waterdrop/producer/sync.rb

@@ -8,7 +8,7 @@ module WaterDrop
       #
       # @param message [Hash] hash that complies with the {Contracts::Message} contract
       #
-      # @return [Rdkafka::Producer::DeliveryHandle] delivery report
+      # @return [Rdkafka::Producer::DeliveryReport] delivery report
       #
       # @raise [Rdkafka::RdkafkaError] When adding the message to rdkafka's queue failed
       # @raise [Rdkafka::Producer::WaitTimeoutError] When the timeout has been reached and the
@@ -20,7 +20,7 @@ module WaterDrop
         validate_message!(message)
 
         @monitor.instrument(
-          'message.produced_sync',
+          "message.produced_sync",
           producer_id: id,
           message: message
         ) do
@@ -33,11 +33,11 @@ module WaterDrop
           raise Errors::ProduceError, e.inspect
         rescue Errors::ProduceError => ex
           @monitor.instrument(
-            'error.occurred',
+            "error.occurred",
             producer_id: id,
             message: message,
             error: ex,
-            type: 'message.produce_sync'
+            type: "message.produce_sync"
           )
 
           raise ex
@@ -49,13 +49,20 @@ module WaterDrop
       # @param messages [Array<Hash>] array with messages that comply with the
       #   {Contracts::Message} contract
       #
-      # @return [Array<Rdkafka::Producer::DeliveryHandle>] delivery reports
+      # @return [Array<Rdkafka::Producer::DeliveryHandle>] delivery handles for messages that were
+      #   sent (can be used to verify offset, partition, etc via `#create_result`)
       #
       # @raise [Rdkafka::RdkafkaError] When adding the messages to rdkafka's queue failed
       # @raise [Rdkafka::Producer::WaitTimeoutError] When the timeout has been reached and some
       #   handles are still pending
       # @raise [Errors::MessageInvalidError] When any of the provided messages details are invalid
       #   and the message could not be sent to Kafka
+      #
+      # @note We return delivery handles instead of delivery reports to ensure consistent return
+      #   types for both success and error flows. In case of partial failures (e.g., mid-batch
+      #   errors), returning handles guarantees the same type in the `dispatched` collection,
+      #   allowing uniform error handling. Each handle is in its final state after this method
+      #   returns, so you can call `handle.create_result` to obtain the delivery report if needed.
       def produce_many_sync(messages)
         messages = middleware.run_many(messages)
         messages.each { |message| validate_message!(message) }
@@ -63,7 +70,7 @@ module WaterDrop
         dispatched = []
         inline_error = nil
 
-        @monitor.instrument('messages.produced_sync', producer_id: id, messages: messages) do
+        @monitor.instrument("messages.produced_sync", producer_id: id, messages: messages) do
           # While most of the librdkafka errors are async and not inline, there are some like
           # buffer overflow that can leak in during the `#produce` itself. When this happens, we
           # still (since it's a sync mode) need to wait on deliveries of things that were
@@ -99,7 +106,7 @@ module WaterDrop
         re_raised = Errors::ProduceManyError.new(dispatched, e.inspect)
 
         @monitor.instrument(
-          'error.occurred',
+          "error.occurred",
           producer_id: id,
           messages: messages,
           # If it is a transactional producer nothing was successfully dispatched on error, thus
@@ -108,7 +115,7 @@ module WaterDrop
           # isolation level.
           dispatched: transactional? ? EMPTY_ARRAY : dispatched,
           error: re_raised,
-          type: 'messages.produce_many_sync'
+          type: "messages.produce_many_sync"
         )
 
         raise re_raised

data/lib/waterdrop/producer/testing.rb

@@ -106,7 +106,7 @@ module WaterDrop
         return if client.respond_to?(:trigger_test_fatal_error)
 
         # Require the rdkafka testing module if not already loaded
-        require 'rdkafka/producer/testing' unless defined?(::Rdkafka::Testing)
+        require "rdkafka/producer/testing" unless defined?(::Rdkafka::Testing)
 
         client.singleton_class.include(::Rdkafka::Testing)
       end

data/lib/waterdrop/producer/transactions.rb

@@ -80,7 +80,7 @@ module WaterDrop
               if !e && !finished
                 raise(
                   Errors::EarlyTransactionExitNotAllowedError,
-                  <<~ERROR_MSG.tr("\n", ' ')
+                  <<~ERROR_MSG.tr("\n", " ")
                     Using `return`, `break` or `throw` to exit a transaction block is not allowed.
                     If the `throw` came from `Timeout.timeout(duration)`, pass an exception class as
                     a second argument so it doesn't use `throw` to abort its block.
@@ -110,7 +110,7 @@ module WaterDrop
                   client.abort_transaction
                 end
               end
-            rescue StandardError => e
+            rescue => e
               # If something from rdkafka leaks here, it means there was a non-retryable error that
               # bubbled up. In such cases if we should, we do reload the underling client
               transactional_reload_client_if_needed(e)
@@ -134,7 +134,7 @@ module WaterDrop
       def transactional?
         return @transactional unless @transactional.nil?
 
-        @transactional = config.kafka.to_h.key?(:'transactional.id')
+        @transactional = config.kafka.to_h.key?(:"transactional.id")
       end
 
       # Checks if we can still retry reloading after a transactional fatal error
@@ -229,7 +229,7 @@ module WaterDrop
         do_retry = e.retryable? && attempt < config.max_attempts_on_transaction_command
 
         @monitor.instrument(
-          'error.occurred',
+          "error.occurred",
           producer_id: id,
           caller: self,
           error: e,
@@ -299,7 +299,7 @@ module WaterDrop
           # Users can subscribe to this event and modify event[:caller].config.kafka to change
           # producer config. This is useful for escaping fencing loops by changing transactional.id
           @monitor.instrument(
-            'producer.reload',
+            "producer.reload",
             producer_id: id,
             error: rd_error,
             attempt: @transaction_fatal_error_attempts,
@@ -311,7 +311,7 @@ module WaterDrop
           @transactional = nil
 
           @monitor.instrument(
-            'producer.reloaded',
+            "producer.reloaded",
             producer_id: id,
             attempt: @transaction_fatal_error_attempts
           ) do

data/lib/waterdrop/producer.rb

@@ -62,6 +62,8 @@ module WaterDrop
       @closing_thread_id = nil
       @idempotent = nil
       @transactional = nil
+      @fd_polling = nil
+      @poller = nil
       @idempotent_fatal_error_attempts = 0
       @transaction_fatal_error_attempts = 0
 
@@ -70,7 +72,7 @@ module WaterDrop
 
       # Instrument producer creation for global listeners
       class_monitor.instrument(
-        'producer.created',
+        "producer.created",
         producer: self,
         producer_id: @id
       )
@@ -85,9 +87,9 @@ module WaterDrop
       raise Errors::ProducerAlreadyConfiguredError, id unless @status.initial?
 
       @config = Config
-                .new
-                .setup(...)
-                .config
+        .new
+        .setup(...)
+        .config
 
       @id = @config.id
       @monitor = @config.monitor
@@ -99,7 +101,7 @@ module WaterDrop
 
         # Instrument producer configuration for global listeners
         class_monitor.instrument(
-          'producer.configured',
+          "producer.configured",
           producer: self,
           producer_id: @id,
           config: @config
@@ -121,7 +123,7 @@ module WaterDrop
 
       # Instrument producer configuration for global listeners
       class_monitor.instrument(
-        'producer.configured',
+        "producer.configured",
         producer: self,
         producer_id: @id,
         config: @config
@@ -166,12 +168,47 @@ module WaterDrop
         @client = Builder.new.call(self, @config)
 
         @status.connected!
-        @monitor.instrument('producer.connected', producer_id: id)
+        @monitor.instrument("producer.connected", producer_id: id)
       end
 
       @client
     end
 
+    # Returns the number of messages in the librdkafka producer queue.
+    #
+    # This count includes:
+    # - Messages waiting to be sent to the broker
+    # - Messages currently in-flight (being transmitted)
+    # - Delivery reports waiting to be processed
+    #
+    # @return [Integer] the number of pending messages in the rdkafka queue, or 0 if the
+    #   producer is not connected
+    #
+    # @note This only counts messages in the rdkafka queue, not the internal WaterDrop buffer.
+    #   To get the internal buffer count, use `messages.size`.
+    #
+    # @note Returns 0 when the producer is not connected as there cannot be any pending
+    #   messages if we haven't connected.
+    #
+    # @example Check pending messages
+    #   producer.queue_size #=> 42
+    #
+    # @example Check total pending work (buffer + rdkafka queue)
+    #   internal_buffer = producer.messages.size
+    #   rdkafka_queue = producer.queue_size
+    #   total_pending = internal_buffer + rdkafka_queue
+    def queue_size
+      return 0 unless @status.connected?
+
+      @connecting_mutex.synchronize do
+        return 0 unless @client
+
+        @client.queue_size
+      end
+    end
+
+    alias_method :queue_length, :queue_size
+
     # Fetches and caches the partition count of a topic
     #
     # @param topic [String] topic for which we want to get the number of partitions
@@ -189,7 +226,7 @@ module WaterDrop
     #   purge the internal WaterDrop buffer but will also purge the librdkafka queue as well as
     #   will cancel any outgoing messages dispatches.
     def purge
-      @monitor.instrument('buffer.purged', producer_id: id) do
+      @monitor.instrument("buffer.purged", producer_id: id) do
         @buffer_mutex.synchronize do
           @messages = []
         end
@@ -218,7 +255,7 @@ module WaterDrop
       Variant.new(self, **args)
     end
 
-    alias variant with
+    alias_method :variant, :with
 
     # Returns and caches the middleware object that may be used
     # @return [WaterDrop::Producer::Middleware]
@@ -261,9 +298,12 @@ module WaterDrop
             return false unless @operations_in_progress.value.zero?
 
             @status.disconnecting!
-            @monitor.instrument('producer.disconnecting', producer_id: id)
+            @monitor.instrument("producer.disconnecting", producer_id: id)
+
+            @monitor.instrument("producer.disconnected", producer_id: id) do
+              # Unregister from poller before closing if fiber polling is enabled
+              unregister_from_poller
 
-            @monitor.instrument('producer.disconnected', producer_id: id) do
               # Close the client
               @client.close
               @client = nil
@@ -298,6 +338,16 @@ module WaterDrop
     # @param force [Boolean] should we force closing even with outstanding messages after the
     #   max wait timeout
     def close(force: false)
+      # When closing from within the FD poller thread (e.g., from a callback like
+      # message.acknowledged or error.occurred), we must delegate to a background thread.
+      # Close performs flush which waits for delivery reports, but delivery reports require
+      # the poller to poll. Since we're ON the poller thread inside a callback, this would
+      # deadlock. Spawning a thread allows the callback to return, letting the poller continue.
+      if fd_polling? && poller.in_poller_thread?
+        Thread.new { close(force: force) }
+        return
+      end
+
       # If we already own the transactional mutex, it means we are inside of a transaction and
       # it should not we allowed to close the producer in such a case.
       if @transaction_mutex.locked? && @transaction_mutex.owned?
@@ -311,11 +361,11 @@ module WaterDrop
           return unless @status.active?
 
           @monitor.instrument(
-            'producer.closed',
+            "producer.closed",
             producer_id: id
           ) do
             @status.closing!
-            @monitor.instrument('producer.closing', producer_id: id)
+            @monitor.instrument("producer.closing", producer_id: id)
 
             # No need for auto-gc if everything got closed by us
             # This should be used only in case a producer was not closed properly and forgotten
@@ -355,6 +405,9 @@ module WaterDrop
                 purge if force
               end
 
+              # Unregister from poller before closing if fiber polling is enabled
+              unregister_from_poller
+
               @client.close
 
               @client = nil
@@ -391,20 +444,38 @@ module WaterDrop
           @buffer_mutex.unlock
         end
       else
-        parts << 'buffer_size=busy'
+        parts << "buffer_size=busy"
       end
 
       # Check if client is connected without triggering connection
       parts << if @status.connected?
-                 'connected=true'
-               else
-                 'connected=false'
-               end
+        "connected=true"
+      else
+        "connected=false"
+      end
 
       parts << "operations=#{@operations_in_progress.value}"
-      parts << 'in_transaction=true' if @transaction_mutex.locked?
+      parts << "in_transaction=true" if @transaction_mutex.locked?
+
+      "#<#{self.class.name}:#{format("%#x", object_id)} #{parts.join(" ")}>"
+    end
+
+    # @return [Boolean] true if FD-based polling mode is enabled
+    def fd_polling?
+      return @fd_polling unless @fd_polling.nil?
+      return false unless config
+
+      @fd_polling = config.polling.mode == :fd
+    end
+
+    # Returns the poller instance for this producer
+    # @return [WaterDrop::Polling::Poller] custom poller if configured, otherwise the global
+    #   singleton poller
+    def poller
+      return @poller unless @poller.nil?
+      return nil unless config
 
-      "#<#{self.class.name}:#{format('%#x', object_id)} #{parts.join(' ')}>"
+      @poller = config.polling.poller || Polling::Poller.instance
     end
 
     private
@@ -438,8 +509,7 @@ module WaterDrop
     #   final result and it is an error.
     def wait(handler, raise_response_error: true)
       handler.wait(
-        # rdkafka max_wait_timeout is in seconds and we use ms
-        max_wait_timeout: current_variant.max_wait_timeout / 1_000.0,
+        max_wait_timeout_ms: current_variant.max_wait_timeout,
         raise_response_error: raise_response_error
       )
     end
@@ -479,10 +549,10 @@ module WaterDrop
       end
 
       result = if transactional?
-                 transaction { client.produce(**message) }
-               else
-                 client.produce(**message)
-               end
+        transaction { client.produce(**message) }
+      else
+        client.produce(**message)
+      end
 
       # Reset attempts counter on successful produce
       @idempotent_fatal_error_attempts = 0
@@ -499,10 +569,10 @@ module WaterDrop
 
         # Instrument error.occurred before attempting reload for visibility
         @monitor.instrument(
-          'error.occurred',
+          "error.occurred",
           producer_id: id,
           error: e,
-          type: 'librdkafka.idempotent_fatal_error',
+          type: "librdkafka.idempotent_fatal_error",
           attempt: @idempotent_fatal_error_attempts
         )
 
@@ -526,7 +596,7 @@ module WaterDrop
       # in an infinite loop, effectively hanging the processing
       raise unless monotonic_now - produce_time < @config.wait_timeout_on_queue_full
 
-      label = caller_locations(2, 1)[0].label.split.last.split('#').last
+      label = caller_locations(2, 1)[0].label.split.last.split("#").last
 
       # We use this syntax here because we want to preserve the original `#cause` when we
       # instrument the error and there is no way to manually assign `#cause` value. We want to keep
@@ -546,7 +616,7 @@ module WaterDrop
           # If this type of event happens too often, it may indicate that the buffer settings are
           # not well configured.
           @monitor.instrument(
-            'error.occurred',
+            "error.occurred",
             producer_id: id,
             message: message,
             error: e,
@@ -570,9 +640,22 @@ module WaterDrop
     def reload!
       @client.flush(current_variant.max_wait_timeout)
       purge
+      # Unregister from poller before closing if fiber polling is enabled
+      unregister_from_poller
       @client.close
       @client = nil
       @status.configured!
     end
+
+    # Unregisters this producer from its poller
+    #
+    # @note We only unregister when fd_polling? is true because thread-mode producers never
+    #   register with the Poller. The Poller.unregister method handles unregistered producers
+    #   gracefully, but this guard avoids making unnecessary unregister calls in thread mode.
+    def unregister_from_poller
+      return unless fd_polling?
+
+      poller.unregister(self)
+    end
   end
 end

data/lib/waterdrop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '2.8.14'
+  VERSION = "2.8.16"
 end

data/lib/waterdrop.rb

@@ -1,20 +1,21 @@
 # frozen_string_literal: true
 
 # External components
-require 'delegate'
-require 'forwardable'
-require 'json'
-require 'zeitwerk'
-require 'securerandom'
-require 'karafka-core'
-require 'pathname'
+require "delegate"
+require "forwardable"
+require "json"
+require "zeitwerk"
+require "securerandom"
+require "singleton"
+require "karafka-core"
+require "pathname"
 
 # WaterDrop library
 module WaterDrop
   class << self
     # @return [String] root path of this gem
     def gem_root
-      Pathname.new(File.expand_path('..', __dir__))
+      Pathname.new(File.expand_path("..", __dir__))
     end
 
     # @return [WaterDrop::Instrumentation::ClassMonitor] global monitor for
@@ -34,15 +35,19 @@ module WaterDrop
     end
 
     # @deprecated Use #monitor instead. This method is provided for backward compatibility only.
-    alias instrumentation monitor
+    alias_method :instrumentation, :monitor
   end
 end
 
 loader = Zeitwerk::Loader.for_gem
-loader.inflector.inflect('waterdrop' => 'WaterDrop')
+loader.inflector.inflect("waterdrop" => "WaterDrop")
 # Do not load vendors instrumentation components. Those need to be required manually if needed
 loader.ignore("#{__dir__}/waterdrop/instrumentation/vendors/**/*.rb")
 # Do not load testing components. Those need to be required manually in test environments
 loader.ignore("#{__dir__}/waterdrop/producer/testing.rb")
 loader.setup
 loader.eager_load
+
+# Setup the poller. Even if users do not use it, it is few objects and we prevent any potential
+# race conditions later
+WaterDrop::Polling::Poller.instance