waterdrop 2.8.14 → 2.8.16

data/lib/waterdrop/errors.rb

@@ -56,6 +56,9 @@ module WaterDrop
     # Do not use `break`, `return` or `throw` inside of the transaction blocks
     EarlyTransactionExitNotAllowedError = Class.new(BaseError)
 
+    # Raised when an error occurs in the polling loop
+    PollerError = Class.new(BaseError)
+
     # Raised when during messages producing something bad happened inline
     class ProduceManyError < ProduceError
       attr_reader :dispatched

data/lib/waterdrop/instrumentation/callbacks/delivery.rb

@@ -47,13 +47,13 @@ module WaterDrop
         # This runs from the rdkafka thread, thus we want to safe-guard it and prevent absolute
         # crashes even if the instrumentation code fails. If it would bubble-up, it could crash
         # the rdkafka background thread
-        rescue StandardError => e
+        rescue => e
           @monitor.instrument(
-            'error.occurred',
+            "error.occurred",
             caller: self,
             error: e,
             producer_id: @producer_id,
-            type: 'callbacks.delivery.error'
+            type: "callbacks.delivery.error"
           )
         end
 
@@ -62,7 +62,7 @@ module WaterDrop
         # @param delivery_report [Rdkafka::Producer::DeliveryReport] delivery report
         def instrument_acknowledged(delivery_report)
           @monitor.instrument(
-            'message.acknowledged',
+            "message.acknowledged",
             caller: self,
             producer_id: @producer_id,
             offset: delivery_report.offset,
@@ -76,7 +76,7 @@ module WaterDrop
         # @param delivery_report [Rdkafka::Producer::DeliveryReport] delivery report
         def instrument_purged(delivery_report)
           @monitor.instrument(
-            'message.purged',
+            "message.purged",
             caller: self,
             error: build_error(delivery_report),
             producer_id: @producer_id,
@@ -85,14 +85,14 @@ module WaterDrop
             topic: delivery_report.topic_name,
             delivery_report: delivery_report,
             label: delivery_report.label,
-            type: 'librdkafka.dispatch_error'
+            type: "librdkafka.dispatch_error"
           )
         end
 
         # @param delivery_report [Rdkafka::Producer::DeliveryReport] delivery report
         def instrument_error(delivery_report)
           @monitor.instrument(
-            'error.occurred',
+            "error.occurred",
             caller: self,
             error: build_error(delivery_report),
             producer_id: @producer_id,
@@ -101,7 +101,7 @@ module WaterDrop
             topic: delivery_report.topic_name,
             delivery_report: delivery_report,
             label: delivery_report.label,
-            type: 'librdkafka.dispatch_error'
+            type: "librdkafka.dispatch_error"
           )
         end
 

data/lib/waterdrop/instrumentation/callbacks/error.rb

@@ -26,22 +26,22 @@ module WaterDrop
           return unless @client_name == client_name
 
           @monitor.instrument(
-            'error.occurred',
+            "error.occurred",
             caller: self,
             error: error,
             producer_id: @producer_id,
-            type: 'librdkafka.error'
+            type: "librdkafka.error"
           )
         # This runs from the rdkafka thread, thus we want to safe-guard it and prevent absolute
         # crashes even if the instrumentation code fails. If it would bubble-up, it could crash
         # the rdkafka background thread
-        rescue StandardError => e
+        rescue => e
           @monitor.instrument(
-            'error.occurred',
+            "error.occurred",
             caller: self,
             error: e,
             producer_id: @producer_id,
-            type: 'callbacks.error.error'
+            type: "callbacks.error.error"
           )
         end
       end

data/lib/waterdrop/instrumentation/callbacks/oauthbearer_token_refresh.rb

@@ -27,20 +27,20 @@ module WaterDrop
           return unless @bearer.name == bearer_name
 
           @monitor.instrument(
-            'oauthbearer.token_refresh',
+            "oauthbearer.token_refresh",
             bearer: @bearer,
             caller: self
           )
         # This runs from the rdkafka thread, thus we want to safe-guard it and prevent absolute
         # crashes even if the instrumentation code fails. If it would bubble-up, it could crash
         # the rdkafka background thread
-        rescue StandardError => e
+        rescue => e
           @monitor.instrument(
-            'error.occurred',
+            "error.occurred",
             caller: self,
             error: e,
             producer_id: @producer_id,
-            type: 'callbacks.oauthbearer_token_refresh.error'
+            type: "callbacks.oauthbearer_token_refresh.error"
           )
         end
       end

data/lib/waterdrop/instrumentation/callbacks/statistics.rb

@@ -27,25 +27,38 @@ module WaterDrop
           # rdkafka does not have per-instance statistics hook, thus we need to make sure that we
           # emit only stats that are related to current producer. Otherwise we would emit all of
           # all the time.
-          return unless @client_name == statistics['name']
+          return unless @client_name == statistics["name"]
+
+          # Skip if no one is listening. We check on each emission to support late subscribers
+          # since statistics are emitted every 5 seconds, this check is cheap enough
+          return unless listening?
 
           @monitor.instrument(
-            'statistics.emitted',
+            "statistics.emitted",
             producer_id: @producer_id,
             statistics: @statistics_decorator.call(statistics)
           )
         # This runs from the rdkafka thread, thus we want to safe-guard it and prevent absolute
         # crashes even if the instrumentation code fails. If it would bubble-up, it could crash
         # the rdkafka background thread
-        rescue StandardError => e
+        rescue => e
           @monitor.instrument(
-            'error.occurred',
+            "error.occurred",
             caller: self,
             error: e,
             producer_id: @producer_id,
-            type: 'callbacks.statistics.error'
+            type: "callbacks.statistics.error"
           )
         end
+
+        private
+
+        # Check if anyone is listening to statistics events
+        # @return [Boolean] true if there are listeners
+        def listening?
+          listeners = @monitor.listeners["statistics.emitted"]
+          listeners && !listeners.empty?
+        end
       end
     end
   end

data/lib/waterdrop/instrumentation/idle_disconnector_listener.rb

@@ -63,7 +63,7 @@ module WaterDrop
       # Handles statistics.emitted events to monitor message transmission activity
       # @param statistics [Hash] producer librdkafka statistics
       def call(statistics)
-        current_txmsgs = statistics.fetch('txmsgs', 0)
+        current_txmsgs = statistics.fetch("txmsgs", 0)
         current_time = monotonic_now
 
         # Update activity if messages changed
@@ -86,12 +86,12 @@ module WaterDrop
           # thread creation
           Thread.new do
             @producer.disconnect
-          rescue StandardError => e
+          rescue => e
             @producer.monitor.instrument(
-              'error.occurred',
+              "error.occurred",
               producer_id: @producer.id,
               error: e,
-              type: 'producer.disconnect.error'
+              type: "producer.disconnect.error"
             )
           end
         end

data/lib/waterdrop/instrumentation/logger_listener.rb

@@ -117,36 +117,36 @@ module WaterDrop
 
       # @param event [Dry::Events::Event] event that happened with the details
       def on_buffer_purged(event)
-        info(event, 'Successfully purging buffer')
+        info(event, "Successfully purging buffer")
       end
 
       # @param event [Dry::Events::Event] event that happened with the details
       def on_producer_closing(event)
-        info(event, 'Closing producer')
+        info(event, "Closing producer")
       end
 
       # @param event [Dry::Events::Event] event that happened with the details
       # @note While this says "Closing producer", it produces a nice message with time taken:
       #   "Closing producer took 12 ms" indicating it happened in the past.
       def on_producer_closed(event)
-        info(event, 'Closing producer')
+        info(event, "Closing producer")
       end
 
       # @param event [Dry::Events::Event] event that happened with the details
       def on_producer_disconnecting(event)
-        info(event, 'Disconnecting producer')
+        info(event, "Disconnecting producer")
       end
 
       # @param event [Dry::Events::Event] event that happened with the details
       # @note While this says "Disconnecting producer", it produces a nice message with time taken:
       #   "Disconnecting producer took 5 ms" indicating it happened in the past.
       def on_producer_disconnected(event)
-        info(event, 'Disconnected producer')
+        info(event, "Disconnected producer")
       end
 
       # @param event [Dry::Events::Event] event that happened with the details
       def on_producer_reloaded(event)
-        info(event, 'Producer successfully reloaded')
+        info(event, "Producer successfully reloaded")
       end
 
       # @param event [Dry::Events::Event] event that happened with the error details
@@ -159,17 +159,17 @@ module WaterDrop
 
       # @param event [Dry::Events::Event] event that happened with the details
       def on_transaction_started(event)
-        info(event, 'Starting transaction')
+        info(event, "Starting transaction")
       end
 
       # @param event [Dry::Events::Event] event that happened with the details
       def on_transaction_aborted(event)
-        info(event, 'Aborting transaction')
+        info(event, "Aborting transaction")
       end
 
       # @param event [Dry::Events::Event] event that happened with the details
       def on_transaction_committed(event)
-        info(event, 'Committing transaction')
+        info(event, "Committing transaction")
       end
 
       # @param event [Dry::Events::Event] event that happened with the details
@@ -188,7 +188,7 @@ module WaterDrop
 
       # @param event [Dry::Events::Event] event that happened with the details
       def on_transaction_finished(event)
-        info(event, 'Processing transaction')
+        info(event, "Processing transaction")
       end
 
       private

data/lib/waterdrop/instrumentation/notifications.rb

@@ -37,6 +37,9 @@ module WaterDrop
         buffer.flushed_sync
         buffer.purged
 
+        poller.producer_registered
+        poller.producer_unregistered
+
         statistics.emitted
 
         error.occurred

data/lib/waterdrop/instrumentation/vendors/datadog/metrics_listener.rb

@@ -20,7 +20,7 @@ module WaterDrop
           RdKafkaMetric = Struct.new(:type, :scope, :name, :key_location)
 
           # Namespace under which the DD metrics should be published
-          setting :namespace, default: 'waterdrop'
+          setting :namespace, default: "waterdrop"
 
           # Datadog client that we should use to publish the metrics
           setting :client
@@ -35,19 +35,19 @@ module WaterDrop
           # Note, that the once with `_d` come from WaterDrop, not rdkafka or Kafka
           setting :rd_kafka_metrics, default: [
             # Client metrics
-            RdKafkaMetric.new(:count, :root, 'calls', 'tx_d'),
-            RdKafkaMetric.new(:histogram, :root, 'queue.size', 'msg_cnt'),
+            RdKafkaMetric.new(:count, :root, "calls", "tx_d"),
+            RdKafkaMetric.new(:histogram, :root, "queue.size", "msg_cnt"),
 
             # Broker metrics
-            RdKafkaMetric.new(:count, :brokers, 'deliver.attempts', 'txretries_d'),
-            RdKafkaMetric.new(:count, :brokers, 'deliver.errors', 'txerrs_d'),
-            RdKafkaMetric.new(:count, :brokers, 'receive.errors', 'rxerrs_d'),
-            RdKafkaMetric.new(:gauge, :brokers, 'queue.latency.avg', %w[outbuf_latency avg]),
-            RdKafkaMetric.new(:gauge, :brokers, 'queue.latency.p95', %w[outbuf_latency p95]),
-            RdKafkaMetric.new(:gauge, :brokers, 'queue.latency.p99', %w[outbuf_latency p99]),
-            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.avg', %w[rtt avg]),
-            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.p95', %w[rtt p95]),
-            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.p99', %w[rtt p99])
+            RdKafkaMetric.new(:count, :brokers, "deliver.attempts", "txretries_d"),
+            RdKafkaMetric.new(:count, :brokers, "deliver.errors", "txerrs_d"),
+            RdKafkaMetric.new(:count, :brokers, "receive.errors", "rxerrs_d"),
+            RdKafkaMetric.new(:gauge, :brokers, "queue.latency.avg", %w[outbuf_latency avg]),
+            RdKafkaMetric.new(:gauge, :brokers, "queue.latency.p95", %w[outbuf_latency p95]),
+            RdKafkaMetric.new(:gauge, :brokers, "queue.latency.p99", %w[outbuf_latency p99]),
+            RdKafkaMetric.new(:gauge, :brokers, "network.latency.avg", %w[rtt avg]),
+            RdKafkaMetric.new(:gauge, :brokers, "network.latency.p95", %w[rtt p95]),
+            RdKafkaMetric.new(:gauge, :brokers, "network.latency.p99", %w[rtt p99])
           ].freeze
 
           configure
@@ -78,13 +78,13 @@ module WaterDrop
           #
           # @param _event [Karafka::Core::Monitoring::Event]
           def on_error_occurred(_event)
-            count('error_occurred', 1, tags: default_tags)
+            count("error_occurred", 1, tags: default_tags)
           end
 
           # Increases acknowledged messages counter
           # @param _event [Karafka::Core::Monitoring::Event]
           def on_message_acknowledged(_event)
-            increment('acknowledged', tags: default_tags)
+            increment("acknowledged", tags: default_tags)
           end
 
           %i[
@@ -216,26 +216,26 @@ module WaterDrop
                 tags: default_tags
               )
             when :brokers
-              statistics.fetch('brokers').each_value do |broker_statistics|
+              statistics.fetch("brokers").each_value do |broker_statistics|
                 # Skip bootstrap nodes
                 # Bootstrap nodes have nodeid -1, other nodes have positive
                 # node ids
-                next if broker_statistics['nodeid'] == -1
+                next if broker_statistics["nodeid"] == -1
 
                 public_send(
                   metric.type,
                   metric.name,
                   broker_statistics.dig(*metric.key_location),
-                  tags: default_tags + ["broker:#{broker_statistics['nodename']}"]
+                  tags: default_tags + ["broker:#{broker_statistics["nodename"]}"]
                 )
               end
             when :topics
-              statistics.fetch('topics').each_value do |topic_statistics|
+              statistics.fetch("topics").each_value do |topic_statistics|
                 public_send(
                   metric.type,
                   metric.name,
                   topic_statistics.dig(*metric.key_location),
-                  tags: default_tags + ["topic:#{topic_statistics['topic']}"]
+                  tags: default_tags + ["topic:#{topic_statistics["topic"]}"]
                 )
               end
             else

data/lib/waterdrop/polling/config.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+# WaterDrop main module
+module WaterDrop
+  # Namespace for FD-based polling components
+  module Polling
+    # Configuration for the global FD poller singleton
+    # These settings apply to all producers using FD polling mode
+    #
+    # @example Configure before creating any producers
+    #   WaterDrop::Polling::Config.setup do |config|
+    #     config.thread_priority = -1
+    #     config.poll_timeout = 500
+    #   end
+    class Config
+      extend ::Karafka::Core::Configurable
+
+      # Ruby thread priority for the poller thread
+      # Valid range: -3 to 3 (Ruby's thread priority range)
+      # Higher values = higher priority
+      setting :thread_priority, default: 0
+
+      # IO.select timeout in milliseconds
+      # Controls how often periodic polling happens when no FD events occur
+      # Lower values = more responsive OAuth/stats callbacks but higher CPU
+      setting :poll_timeout, default: 1_000
+
+      # Initial backoff delay in milliseconds after a polling error
+      setting :backoff_min, default: 100
+
+      # Maximum backoff delay in milliseconds after repeated errors
+      # Backoff doubles on each consecutive error up to this limit
+      setting :backoff_max, default: 30_000
+
+      class << self
+        # Configures the poller settings
+        # @yield [config] Configuration block
+        # @yieldparam config [Karafka::Core::Configurable::Node] config node
+        def setup
+          configure do |config|
+            yield(config)
+          end
+
+          Contracts::PollerConfig.new.validate!(
+            self.config.to_h,
+            Errors::ConfigurationInvalidError
+          )
+        end
+      end
+    end
+  end
+end

data/lib/waterdrop/polling/latch.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Polling
+    # A thread-safe latch for synchronizing producer close operations
+    #
+    # When a producer is closed, two threads are involved:
+    # 1. The caller thread (user code calling producer.close)
+    # 2. The poller thread (background thread running IO.select)
+    #
+    # The close sequence:
+    # 1. Caller calls producer.close -> unregister_from_poller -> Poller#unregister
+    # 2. Poller#unregister signals via control pipe and calls state.wait_for_close (blocks on latch)
+    # 3. Poller thread receives control signal, drains queue, calls state.close
+    # 4. state.close releases the latch via release!
+    # 5. Caller's wait_for_close returns, unregister completes
+    #
+    # This ensures the producer is fully drained and removed from the poller
+    # before returning control to the caller, preventing race conditions.
+    class Latch
+      def initialize
+        @mutex = Mutex.new
+        @cv = ConditionVariable.new
+        @released = false
+      end
+
+      # Releases the latch and wakes any waiting threads
+      def release!
+        @mutex.synchronize do
+          @released = true
+          @cv.broadcast
+        end
+      end
+
+      # Waits until the latch is released
+      # Returns immediately if already released
+      def wait
+        @mutex.synchronize do
+          @cv.wait(@mutex) until @released
+        end
+      end
+
+      # @return [Boolean] whether the latch has been released
+      def released?
+        @released
+      end
+    end
+  end
+end