waterdrop 2.8.6 → 2.8.8.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 96b89ee79adb2be90c97e5e87f79d79ad86ceb7d3652c0220d32030e605afadf
-  data.tar.gz: fbdc38cc1c913e3dd433fa4448848f9f3f3799552d2c18741dd207849f59750d
+  metadata.gz: 51626a5ebbcd31ccb011d37675d29da824a0066993cf1a5e0b6117bc49c03c3e
+  data.tar.gz: 9d3755db2b6daf1fd54bed8a526de70237d60ddda4660a1abeab778a5b92183d
 SHA512:
-  metadata.gz: 253cdaf4f5ba5e0f92e723d0a8214dd36b05a9b11986c005a63399bce38f193a90922281cf150f434ea4f0aaa6e6950dbb02a7df2e4134e0508ed474b0a68c04
-  data.tar.gz: 07043a71104f7a818dcb0a4261ee559b62aadb37ad28696ae5ca6d8c0c66c20de5f7d7ad3ab7bfd68ef0b9f91b1f55d67cc1319e37c009a0023beea6fa49905d
+  metadata.gz: 9060a4f68526fb4d3522fd58f69eb7a11c8ff58fcb3bc2a7d140715693bf4f4d4342bb2f79a8d7b75b7b4ad3178a3fa1d5732cdf1add8395b4f93d4abf4daee6
+  data.tar.gz: 236a114d76f18fd84a9802583dd32eb4494298d5ec0e9a11677430e474b71639561a98dbf50766998ea994fdb26f136bdc5ee0e1c91ec0553b7e2d81bb76e3e7

data/.github/workflows/ci.yml

@@ -6,9 +6,7 @@ concurrency:
 
 on:
   pull_request:
-    branches: [ main, master ]
-  push:
-    branches: [ main, master ]
+    branches: [ master ]
   schedule:
     - cron:  '0 1 * * *'
 
@@ -30,7 +28,6 @@ jobs:
           - '3.4'
           - '3.3'
           - '3.2'
-          - '3.1'
         force_ruby_platform:
           - true
           - false
@@ -48,7 +45,7 @@ jobs:
       - name: Remove platform-specific ffi entries for Ruby previews
         if: contains(matrix.ruby, '3.5')
         run: |
-          ruby -i -ne 'puts $_ unless /^\s*ffi \(.*-.*\)$/' Gemfile.lock
+          sed -i '/^\s*ffi (.*-.*)$/d' Gemfile.lock
 
       - name: Set up Ruby
         uses: ruby/setup-ruby@2a7b30092b0caf9c046252510f9273b4875f3db9 # v1.254.0
@@ -56,6 +53,7 @@ jobs:
           ruby-version: ${{matrix.ruby}}
           bundler-cache: true
           bundler: 'latest'
+          self-hosted: false
 
       - name: Run Kafka with Docker Compose
         run: |
@@ -91,6 +89,7 @@ jobs:
 
   diffend:
     timeout-minutes: 5
+
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -102,6 +101,8 @@ jobs:
         uses: ruby/setup-ruby@2a7b30092b0caf9c046252510f9273b4875f3db9 # v1.254.0
         with:
           ruby-version: 3.4
+          self-hosted: false
+
       - name: Install latest bundler
         run: gem install bundler --no-document
       - name: Install Diffend plugin
@@ -132,3 +133,20 @@ jobs:
           fi
       - name: Run Coditsu
         run: ./coditsu_script.sh
+
+  ci-success:
+    name: CI Success
+    runs-on: ubuntu-latest
+    if: always()
+    needs:
+      - diffend
+      - coditsu
+      - specs
+    steps:
+      - name: Check all jobs passed
+        if: |
+          contains(needs.*.result, 'failure') ||
+          contains(needs.*.result, 'cancelled') ||
+          contains(needs.*.result, 'skipped')
+        run: exit 1
+      - run: echo "All CI checks passed!"

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 # WaterDrop changelog
 
+## 2.8.8 (Unreleased)
+- [Feature] Add `WaterDrop::ConnectionPool` for efficient connection pooling using the proven `connection_pool` gem.
+- [Feature] Add `WaterDrop.instrumentation` class-level instrumentation for producer lifecycle events. This allows external libraries to subscribe to `producer.created` and `producer.configured` events without needing producer instance references, enabling middleware injection and configuration by libraries like Datadog tracing.
+- [Change] Remove Ruby `3.1` specs according to the EOL schedule.
+
+## 2.8.7 (2025-09-02)
+- [Enhancement] Disable Nagle algorithm by default (improves latency / aligned with librdkafka)
+- [Change] Normalize how libs and dependencies are required (no functional change for the end user)
+
 ## 2.8.6 (2025-08-18)
 - [Feature] Add `idle_disconnect_timeout` config option to automatically disconnect idle producers after a configurable timeout period.
 - [Feature] Add support for [async](https://github.com/socketry/async) gems ecosystem with proper fiber yielding during blocking operations.

data/Gemfile

@@ -14,6 +14,7 @@ group :development do
 end
 
 group :test do
+  gem 'connection_pool'
   gem 'ostruct'
   gem 'rspec'
   gem 'simplecov'

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.8.6)
+    waterdrop (2.8.8.rc1)
       karafka-core (>= 2.4.9, < 3.0.0)
       karafka-rdkafka (>= 0.20.0)
       zeitwerk (~> 2.3)
@@ -10,6 +10,7 @@ GEM
   remote: https://rubygems.org/
   specs:
     byebug (12.0.0)
+    connection_pool (2.5.4)
     diff-lcs (1.6.2)
     docile (1.4.1)
     ffi (1.17.2)
@@ -23,26 +24,37 @@ GEM
     ffi (1.17.2-x86_64-darwin)
     ffi (1.17.2-x86_64-linux-gnu)
     ffi (1.17.2-x86_64-linux-musl)
-    karafka-core (2.5.2)
-      karafka-rdkafka (>= 0.19.2, < 0.21.0)
+    json (2.13.2)
+    karafka-core (2.5.6)
+      karafka-rdkafka (>= 0.20.0)
       logger (>= 1.6.0)
-    karafka-rdkafka (0.21.0.rc1)
+    karafka-rdkafka (0.21.0)
+      ffi (~> 1.15)
+      json (> 2.0)
+      logger
+      mini_portile2 (~> 2.6)
+      rake (> 12)
+    karafka-rdkafka (0.21.0-aarch64-linux-gnu)
       ffi (~> 1.15)
+      json (> 2.0)
       logger
       mini_portile2 (~> 2.6)
       rake (> 12)
-    karafka-rdkafka (0.21.0.rc1-arm64-darwin)
+    karafka-rdkafka (0.21.0-arm64-darwin)
       ffi (~> 1.15)
+      json (> 2.0)
       logger
       mini_portile2 (~> 2.6)
       rake (> 12)
-    karafka-rdkafka (0.21.0.rc1-x86_64-linux-gnu)
+    karafka-rdkafka (0.21.0-x86_64-linux-gnu)
       ffi (~> 1.15)
+      json (> 2.0)
       logger
       mini_portile2 (~> 2.6)
       rake (> 12)
-    karafka-rdkafka (0.21.0.rc1-x86_64-linux-musl)
+    karafka-rdkafka (0.21.0-x86_64-linux-musl)
       ffi (~> 1.15)
+      json (> 2.0)
       logger
       mini_portile2 (~> 2.6)
       rake (> 12)
@@ -87,6 +99,7 @@ PLATFORMS
 
 DEPENDENCIES
   byebug
+  connection_pool
   ostruct
   rspec
   simplecov

data/bin/verify_kafka_warnings

@@ -9,7 +9,7 @@ allowed_patterns=(
 )
 
 # Get all warnings
-warnings=$(docker logs --since=0 kafka | grep WARN)
+warnings=$(docker logs --since=0 kafka | grep "] WARN ")
 exit_code=0
 
 while IFS= read -r line; do

data/lib/waterdrop/config.rb

@@ -17,7 +17,10 @@ module WaterDrop
       # sync delivery
       'message.timeout.ms': 50_000,
       # Must be more or equal to `message.timeout.ms` defaults
-      'transaction.timeout.ms': 55_000
+      'transaction.timeout.ms': 55_000,
+      # Lowers latency. Default in newer librdkafka but we want to make sure it is shipped to
+      # users despite what librdkafka they run on
+      'socket.nagle.disable': true
     }.freeze
 
     private_constant :KAFKA_DEFAULTS

data/lib/waterdrop/connection_pool.rb

@@ -0,0 +1,233 @@
+# frozen_string_literal: true
+
+# WaterDrop library
+module WaterDrop
+  # Connection pool wrapper for WaterDrop producers using the proven connection_pool gem.
+  #
+  # This provides a clean WaterDrop-specific API while leveraging the battle-tested,
+  # connection_pool gem underneath. The wrapper hides the direct usage of the connection_pool
+  # gem and provides WaterDrop-specific configuration.
+  #
+  # @example Basic usage
+  #   pool = WaterDrop::ConnectionPool.new(size: 10) do |config|
+  #     config.kafka = { 'bootstrap.servers': 'localhost:9092' }
+  #     config.deliver = true
+  #   end
+  #
+  #   pool.with do |producer|
+  #     producer.produce_sync(topic: 'events', payload: 'data')
+  #   end
+  #
+  # @example Transactional producers with unique IDs
+  #   pool = WaterDrop::ConnectionPool.new(size: 5) do |config, index|
+  #     config.kafka = {
+  #       'bootstrap.servers': 'localhost:9092',
+  #       'transactional.id': "my-app-#{index}"
+  #     }
+  #   end
+  #
+  # @example Global connection pool
+  #   WaterDrop::ConnectionPool.setup(size: 20) do |config|
+  #     config.kafka = { 'bootstrap.servers': ENV['KAFKA_BROKERS'] }
+  #   end
+  #
+  #   WaterDrop::ConnectionPool.with do |producer|
+  #     producer.produce_async(topic: 'events', payload: 'data')
+  #   end
+  class ConnectionPool
+    # Delegate key methods to underlying connection pool
+    extend Forwardable
+
+    def_delegators :@pool, :with, :size, :available
+
+    class << self
+      # Global connection pool instance
+      attr_accessor :default_pool
+
+      # Sets up a global connection pool
+      #
+      # @param size [Integer] Pool size (default: 5)
+      # @param timeout [Numeric] Connection timeout in seconds (default: 5)
+      # @param producer_config [Proc] Block to configure each producer in the pool
+      # @yield [config, index] Block to configure each producer in the pool, receives config and
+      #   pool index
+      # @return [ConnectionPool] The configured global pool
+      #
+      # @example Basic setup
+      #   WaterDrop::ConnectionPool.setup(size: 15) do |config|
+      #     config.kafka = { 'bootstrap.servers': ENV['KAFKA_BROKERS'] }
+      #     config.deliver = true
+      #   end
+      #
+      # @example Transactional setup with unique IDs
+      #   WaterDrop::ConnectionPool.setup(size: 5) do |config, index|
+      #     config.kafka = {
+      #       'bootstrap.servers': ENV['KAFKA_BROKERS'],
+      #       'transactional.id': "my-app-#{index}"
+      #     }
+      #   end
+      def setup(size: 5, timeout: 5, &producer_config)
+        ensure_connection_pool_gem!
+
+        @default_pool = new(size: size, timeout: timeout, &producer_config)
+      end
+
+      # Executes a block with a producer from the global pool
+      #
+      # @param block [Proc] Block to execute with a producer
+      # @yield [producer] Producer from the global pool
+      # @return [Object] Result of the block
+      # @raise [RuntimeError] If no global pool is configured
+      #
+      # @example
+      #   WaterDrop::ConnectionPool.with do |producer|
+      #     producer.produce_sync(topic: 'events', payload: 'data')
+      #   end
+      def with(&block)
+        raise 'No global connection pool configured. Call setup first.' unless @default_pool
+
+        @default_pool.with(&block)
+      end
+
+      # Get statistics about the global pool
+      #
+      # @return [Hash, nil] Pool statistics or nil if no global pool
+      def stats
+        return nil unless @default_pool
+
+        {
+          size: @default_pool.size,
+          available: @default_pool.available
+        }
+      end
+
+      # Shutdown the global connection pool
+      def shutdown
+        return unless @default_pool
+
+        @default_pool.shutdown
+        @default_pool = nil
+      end
+
+      # Reload the global connection pool
+      def reload
+        @default_pool&.reload
+      end
+
+      # Check if the global connection pool is active (configured)
+      #
+      # @return [Boolean] true if global pool is configured, false otherwise
+      def active?
+        !@default_pool.nil?
+      end
+
+      private
+
+      # Ensures the connection_pool gem is available (class method)
+      # Only requires it when actually needed (lazy loading)
+      def ensure_connection_pool_gem!
+        return if defined?(::ConnectionPool)
+
+        require 'connection_pool'
+      rescue LoadError
+        raise LoadError, <<~ERROR
+          WaterDrop::ConnectionPool requires the 'connection_pool' gem.
+
+          Add this to your Gemfile:
+              gem 'connection_pool'
+
+          Then run:
+              bundle install
+        ERROR
+      end
+    end
+
+    # Creates a new WaterDrop connection pool
+    #
+    # @param size [Integer] Pool size (default: 5)
+    # @param timeout [Numeric] Connection timeout in seconds (default: 5)
+    # @param producer_config [Proc] Block to configure each producer in the pool
+    # @yield [config, index] Block to configure each producer in the pool, receives config and
+    #   pool index
+    def initialize(size: 5, timeout: 5, &producer_config)
+      self.class.send(:ensure_connection_pool_gem!)
+
+      @producer_config = producer_config
+      @pool_index = 0
+      @pool_mutex = Mutex.new
+
+      @pool = ::ConnectionPool.new(size: size, timeout: timeout) do
+        producer_index = @pool_mutex.synchronize { @pool_index += 1 }
+
+        WaterDrop::Producer.new do |config|
+          if @producer_config.arity == 2
+            @producer_config.call(config, producer_index)
+          else
+            @producer_config.call(config)
+          end
+        end
+      end
+    end
+
+    # Get pool statistics
+    #
+    # @return [Hash] Pool statistics
+    def stats
+      {
+        size: @pool.size,
+        available: @pool.available
+      }
+    end
+
+    # Shutdown the connection pool
+    def shutdown
+      @pool.shutdown do |producer|
+        producer.close! if producer&.status&.active?
+      end
+    end
+
+    # Reload all connections in the pool
+    # Useful for configuration changes or error recovery
+    def reload
+      @pool.reload do |producer|
+        producer.close! if producer&.status&.active?
+      end
+    end
+
+    # Returns the underlying connection_pool instance
+    # This allows access to advanced connection_pool features if needed
+    #
+    # @return [::ConnectionPool] The underlying connection pool
+    attr_reader :pool
+  end
+
+  # Convenience methods on the WaterDrop module for global pool access
+  class << self
+    # Execute a block with a producer from the global connection pool
+    # Only available when connection pool is configured
+    #
+    # @param block [Proc] Block to execute with a producer
+    # @yield [producer] Producer from the global pool
+    # @return [Object] Result of the block
+    #
+    # @example
+    #   WaterDrop.with do |producer|
+    #     producer.produce_sync(topic: 'events', payload: 'data')
+    #   end
+    def with(&block)
+      ConnectionPool.with(&block)
+    end
+
+    # Access the global connection pool
+    #
+    # @return [WaterDrop::ConnectionPool] The global pool
+    #
+    # @example
+    #   WaterDrop.pool.with do |producer|
+    #     producer.produce_async(topic: 'events', payload: 'data')
+    #   end
+    def pool
+      ConnectionPool.default_pool
+    end
+  end
+end

data/lib/waterdrop/instrumentation/class_monitor.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Instrumentation
+    # WaterDrop class-level instrumentation monitor for global events
+    # This monitor only supports class-level lifecycle events, not per-producer events
+    class ClassMonitor < ::Karafka::Core::Monitoring::Monitor
+      # @param notifications_bus [Object] class-level notifications bus
+      # @param namespace [String, nil] namespace for events or nil if no namespace
+      def initialize(
+        notifications_bus = WaterDrop::Instrumentation::ClassNotifications.new,
+        namespace = nil
+      )
+        super(notifications_bus, namespace)
+      end
+    end
+  end
+end

data/lib/waterdrop/instrumentation/class_notifications.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Instrumentation
+    # Class-level notifications for WaterDrop global instrumentation
+    # This only supports events that occur at the class/module level, not per-producer instance
+    class ClassNotifications < ::Karafka::Core::Monitoring::Notifications
+      # List of events that are available at the class level via WaterDrop.instrumentation
+      # These are lifecycle events for producer creation and configuration
+      EVENTS = %w[
+        producer.created
+        producer.configured
+      ].freeze
+
+      # @return [WaterDrop::Instrumentation::ClassNotifications] class-level notification instance
+      def initialize
+        super
+        EVENTS.each { |event| register_event(event) }
+      end
+    end
+  end
+end

data/lib/waterdrop/instrumentation/notifications.rb

@@ -4,7 +4,7 @@ module WaterDrop
   module Instrumentation
     # Instrumented is used to hookup external monitoring services to monitor how WaterDrop works
     class Notifications < ::Karafka::Core::Monitoring::Notifications
-      # List of events that we support in the system and to which a monitor client can hook up
+      # List of events that we support at the instance level (per-producer instrumentation)
       # @note The non-error once support timestamp benchmarking
       EVENTS = %w[
         producer.connected

data/lib/waterdrop/producer/class_monitor.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  class Producer
+    # Module that provides class-level instrumentation capabilities to producers
+    # This allows producers to emit lifecycle events that can be subscribed to at the global level
+    module ClassMonitor
+      private
+
+      # @return [WaterDrop::Instrumentation::ClassMonitor] global class-level monitor for
+      #   instrumentation events
+      def class_monitor
+        WaterDrop.instrumentation
+      end
+    end
+  end
+end

data/lib/waterdrop/producer.rb

@@ -8,6 +8,7 @@ module WaterDrop
     include Async
     include Buffer
     include Transactions
+    include ClassMonitor
     include ::Karafka::Core::Helpers::Time
     include ::Karafka::Core::Taggable
 
@@ -60,6 +61,13 @@ module WaterDrop
       @status = Status.new
       @messages = []
 
+      # Instrument producer creation for global listeners
+      class_monitor.instrument(
+        'producer.created',
+        producer: self,
+        producer_id: @id
+      )
+
       return unless block
 
       setup(&block)
@@ -80,7 +88,19 @@ module WaterDrop
       @contract = Contracts::Message.new(max_payload_size: @config.max_payload_size)
       @default_variant = Variant.new(self, default: true)
 
-      return @status.configured! if @config.idle_disconnect_timeout.zero?
+      if @config.idle_disconnect_timeout.zero?
+        @status.configured!
+
+        # Instrument producer configuration for global listeners
+        class_monitor.instrument(
+          'producer.configured',
+          producer: self,
+          producer_id: @id,
+          config: @config
+        )
+
+        return
+      end
 
       # Setup idle disconnect listener if configured so we preserve tcp connections on rarely
       # used producers
@@ -92,6 +112,14 @@ module WaterDrop
       @monitor.subscribe(disconnector)
 
       @status.configured!
+
+      # Instrument producer configuration for global listeners
+      class_monitor.instrument(
+        'producer.configured',
+        producer: self,
+        producer_id: @id,
+        config: @config
+      )
     end
 
     # @return [Rdkafka::Producer] raw rdkafka producer

data/lib/waterdrop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '2.8.6'
+  VERSION = '2.8.8.rc1'
 end

data/lib/waterdrop.rb

@@ -1,16 +1,13 @@
 # frozen_string_literal: true
 
 # External components
-# delegate should be removed because we don't need it, we just add it because of ruby-kafka
-%w[
-  delegate
-  forwardable
-  json
-  zeitwerk
-  securerandom
-  karafka-core
-  pathname
-].each { |lib| require lib }
+require 'delegate'
+require 'forwardable'
+require 'json'
+require 'zeitwerk'
+require 'securerandom'
+require 'karafka-core'
+require 'pathname'
 
 # WaterDrop library
 module WaterDrop
@@ -19,6 +16,22 @@ module WaterDrop
     def gem_root
       Pathname.new(File.expand_path('..', __dir__))
     end
+
+    # @return [WaterDrop::Instrumentation::ClassMonitor] global instrumentation monitor for
+    #   class-level event subscriptions. This allows external libraries to subscribe to WaterDrop
+    #   lifecycle events without needing producer instance references.
+    #
+    # @note Only supports class-level events (producer.created, producer.configured), not
+    #   instance events
+    #
+    # @example Subscribe to producer creation events
+    #   WaterDrop.instrumentation.subscribe('producer.created') do |event|
+    #     producer = event[:producer]
+    #     # Configure producer or add middleware
+    #   end
+    def instrumentation
+      @instrumentation ||= Instrumentation::ClassMonitor.new
+    end
   end
 end
 

data/waterdrop.gemspec

@@ -27,7 +27,6 @@ Gem::Specification.new do |spec|
   spec.require_paths = %w[lib]
 
   spec.metadata = {
-    'funding_uri' => 'https://karafka.io/#become-pro',
     'homepage_uri' => 'https://karafka.io',
     'changelog_uri' => 'https://karafka.io/docs/Changelog-WaterDrop',
     'bug_tracker_uri' => 'https://github.com/karafka/waterdrop/issues',

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 2.8.6
+  version: 2.8.8.rc1
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -94,6 +94,7 @@ files:
 - lib/waterdrop/clients/dummy.rb
 - lib/waterdrop/clients/rdkafka.rb
 - lib/waterdrop/config.rb
+- lib/waterdrop/connection_pool.rb
 - lib/waterdrop/contracts.rb
 - lib/waterdrop/contracts/config.rb
 - lib/waterdrop/contracts/message.rb
@@ -105,6 +106,8 @@ files:
 - lib/waterdrop/instrumentation/callbacks/error.rb
 - lib/waterdrop/instrumentation/callbacks/oauthbearer_token_refresh.rb
 - lib/waterdrop/instrumentation/callbacks/statistics.rb
+- lib/waterdrop/instrumentation/class_monitor.rb
+- lib/waterdrop/instrumentation/class_notifications.rb
 - lib/waterdrop/instrumentation/idle_disconnector_listener.rb
 - lib/waterdrop/instrumentation/logger_listener.rb
 - lib/waterdrop/instrumentation/monitor.rb
@@ -116,6 +119,7 @@ files:
 - lib/waterdrop/producer/async.rb
 - lib/waterdrop/producer/buffer.rb
 - lib/waterdrop/producer/builder.rb
+- lib/waterdrop/producer/class_monitor.rb
 - lib/waterdrop/producer/status.rb
 - lib/waterdrop/producer/sync.rb
 - lib/waterdrop/producer/transactions.rb
@@ -129,7 +133,6 @@ licenses:
 - LGPL-3.0-only
 - Commercial
 metadata:
-  funding_uri: https://karafka.io/#become-pro
   homepage_uri: https://karafka.io
   changelog_uri: https://karafka.io/docs/Changelog-WaterDrop
   bug_tracker_uri: https://github.com/karafka/waterdrop/issues