waterdrop 2.8.7 → 2.8.8.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c6fd5bbc4f935b3d55794ad4f62970ab9feab3d4ebd100f9394c1bb456b8a9b5
-  data.tar.gz: 00ab23a92637285766a29741324b65422b77aba3522aa812b0f72350e3061585
+  metadata.gz: 51626a5ebbcd31ccb011d37675d29da824a0066993cf1a5e0b6117bc49c03c3e
+  data.tar.gz: 9d3755db2b6daf1fd54bed8a526de70237d60ddda4660a1abeab778a5b92183d
 SHA512:
-  metadata.gz: 0a41a342c8b16e167f1292ea6068647ac100f21f7f632342280a0bf69debcee3e6929f070e63b0fde2c28b85454edc6845d66895376a953c3ccaad48873646df
-  data.tar.gz: 89956ae428a553f91d91a8064954dea2095cb81457fd78f1565e31ec47e31799b823942931a08f1e6f2d0a7f04b7f730d5d1856fe01fd4de9a57055b5e301360
+  metadata.gz: 9060a4f68526fb4d3522fd58f69eb7a11c8ff58fcb3bc2a7d140715693bf4f4d4342bb2f79a8d7b75b7b4ad3178a3fa1d5732cdf1add8395b4f93d4abf4daee6
+  data.tar.gz: 236a114d76f18fd84a9802583dd32eb4494298d5ec0e9a11677430e474b71639561a98dbf50766998ea994fdb26f136bdc5ee0e1c91ec0553b7e2d81bb76e3e7

data/.github/workflows/ci.yml

@@ -28,7 +28,6 @@ jobs:
           - '3.4'
           - '3.3'
           - '3.2'
-          - '3.1'
         force_ruby_platform:
           - true
           - false

data/.github/workflows/trigger-wiki-refresh.yml

@@ -8,14 +8,7 @@ on:
 
 jobs:
   trigger-wiki-refresh:
-    runs-on: >-
-      ${{
-        (github.event_name != 'pull_request' ||
-         github.event.pull_request.head.repo.full_name == github.repository)
-        && fromJSON('["self-hosted", "linux", "x64", "qemu"]')
-        || 'ubuntu-latest'
-      }}
-
+    runs-on: ubuntu-latest
     environment: wiki-trigger
     if: github.repository_owner == 'karafka'
     steps:

data/.github/workflows/verify-action-pins.yml

@@ -5,14 +5,7 @@ on:
       - '.github/workflows/**'
 jobs:
   verify_action_pins:
-    runs-on: >-
-      ${{
-        (github.event_name != 'pull_request' ||
-         github.event.pull_request.head.repo.full_name == github.repository)
-        && fromJSON('["self-hosted", "linux", "x64", "qemu"]')
-        || 'ubuntu-latest'
-      }}
-
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Check SHA pins

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # WaterDrop changelog
 
+## 2.8.8 (Unreleased)
+- [Feature] Add `WaterDrop::ConnectionPool` for efficient connection pooling using the proven `connection_pool` gem.
+- [Feature] Add `WaterDrop.instrumentation` class-level instrumentation for producer lifecycle events. This allows external libraries to subscribe to `producer.created` and `producer.configured` events without needing producer instance references, enabling middleware injection and configuration by libraries like Datadog tracing.
+- [Change] Remove Ruby `3.1` specs according to the EOL schedule.
+
 ## 2.8.7 (2025-09-02)
 - [Enhancement] Disable Nagle algorithm by default (improves latency / aligned with librdkafka)
 - [Change] Normalize how libs and dependencies are required (no functional change for the end user)

data/Gemfile

@@ -14,6 +14,7 @@ group :development do
 end
 
 group :test do
+  gem 'connection_pool'
   gem 'ostruct'
   gem 'rspec'
   gem 'simplecov'

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.8.7)
+    waterdrop (2.8.8.rc1)
       karafka-core (>= 2.4.9, < 3.0.0)
       karafka-rdkafka (>= 0.20.0)
       zeitwerk (~> 2.3)
@@ -10,6 +10,7 @@ GEM
   remote: https://rubygems.org/
   specs:
     byebug (12.0.0)
+    connection_pool (2.5.4)
     diff-lcs (1.6.2)
     docile (1.4.1)
     ffi (1.17.2)
@@ -98,6 +99,7 @@ PLATFORMS
 
 DEPENDENCIES
   byebug
+  connection_pool
   ostruct
   rspec
   simplecov

data/lib/waterdrop/connection_pool.rb

@@ -0,0 +1,233 @@
+# frozen_string_literal: true
+
+# WaterDrop library
+module WaterDrop
+  # Connection pool wrapper for WaterDrop producers using the proven connection_pool gem.
+  #
+  # This provides a clean WaterDrop-specific API while leveraging the battle-tested,
+  # connection_pool gem underneath. The wrapper hides the direct usage of the connection_pool
+  # gem and provides WaterDrop-specific configuration.
+  #
+  # @example Basic usage
+  #   pool = WaterDrop::ConnectionPool.new(size: 10) do |config|
+  #     config.kafka = { 'bootstrap.servers': 'localhost:9092' }
+  #     config.deliver = true
+  #   end
+  #
+  #   pool.with do |producer|
+  #     producer.produce_sync(topic: 'events', payload: 'data')
+  #   end
+  #
+  # @example Transactional producers with unique IDs
+  #   pool = WaterDrop::ConnectionPool.new(size: 5) do |config, index|
+  #     config.kafka = {
+  #       'bootstrap.servers': 'localhost:9092',
+  #       'transactional.id': "my-app-#{index}"
+  #     }
+  #   end
+  #
+  # @example Global connection pool
+  #   WaterDrop::ConnectionPool.setup(size: 20) do |config|
+  #     config.kafka = { 'bootstrap.servers': ENV['KAFKA_BROKERS'] }
+  #   end
+  #
+  #   WaterDrop::ConnectionPool.with do |producer|
+  #     producer.produce_async(topic: 'events', payload: 'data')
+  #   end
+  class ConnectionPool
+    # Delegate key methods to underlying connection pool
+    extend Forwardable
+
+    def_delegators :@pool, :with, :size, :available
+
+    class << self
+      # Global connection pool instance
+      attr_accessor :default_pool
+
+      # Sets up a global connection pool
+      #
+      # @param size [Integer] Pool size (default: 5)
+      # @param timeout [Numeric] Connection timeout in seconds (default: 5)
+      # @param producer_config [Proc] Block to configure each producer in the pool
+      # @yield [config, index] Block to configure each producer in the pool, receives config and
+      #   pool index
+      # @return [ConnectionPool] The configured global pool
+      #
+      # @example Basic setup
+      #   WaterDrop::ConnectionPool.setup(size: 15) do |config|
+      #     config.kafka = { 'bootstrap.servers': ENV['KAFKA_BROKERS'] }
+      #     config.deliver = true
+      #   end
+      #
+      # @example Transactional setup with unique IDs
+      #   WaterDrop::ConnectionPool.setup(size: 5) do |config, index|
+      #     config.kafka = {
+      #       'bootstrap.servers': ENV['KAFKA_BROKERS'],
+      #       'transactional.id': "my-app-#{index}"
+      #     }
+      #   end
+      def setup(size: 5, timeout: 5, &producer_config)
+        ensure_connection_pool_gem!
+
+        @default_pool = new(size: size, timeout: timeout, &producer_config)
+      end
+
+      # Executes a block with a producer from the global pool
+      #
+      # @param block [Proc] Block to execute with a producer
+      # @yield [producer] Producer from the global pool
+      # @return [Object] Result of the block
+      # @raise [RuntimeError] If no global pool is configured
+      #
+      # @example
+      #   WaterDrop::ConnectionPool.with do |producer|
+      #     producer.produce_sync(topic: 'events', payload: 'data')
+      #   end
+      def with(&block)
+        raise 'No global connection pool configured. Call setup first.' unless @default_pool
+
+        @default_pool.with(&block)
+      end
+
+      # Get statistics about the global pool
+      #
+      # @return [Hash, nil] Pool statistics or nil if no global pool
+      def stats
+        return nil unless @default_pool
+
+        {
+          size: @default_pool.size,
+          available: @default_pool.available
+        }
+      end
+
+      # Shutdown the global connection pool
+      def shutdown
+        return unless @default_pool
+
+        @default_pool.shutdown
+        @default_pool = nil
+      end
+
+      # Reload the global connection pool
+      def reload
+        @default_pool&.reload
+      end
+
+      # Check if the global connection pool is active (configured)
+      #
+      # @return [Boolean] true if global pool is configured, false otherwise
+      def active?
+        !@default_pool.nil?
+      end
+
+      private
+
+      # Ensures the connection_pool gem is available (class method)
+      # Only requires it when actually needed (lazy loading)
+      def ensure_connection_pool_gem!
+        return if defined?(::ConnectionPool)
+
+        require 'connection_pool'
+      rescue LoadError
+        raise LoadError, <<~ERROR
+          WaterDrop::ConnectionPool requires the 'connection_pool' gem.
+
+          Add this to your Gemfile:
+              gem 'connection_pool'
+
+          Then run:
+              bundle install
+        ERROR
+      end
+    end
+
+    # Creates a new WaterDrop connection pool
+    #
+    # @param size [Integer] Pool size (default: 5)
+    # @param timeout [Numeric] Connection timeout in seconds (default: 5)
+    # @param producer_config [Proc] Block to configure each producer in the pool
+    # @yield [config, index] Block to configure each producer in the pool, receives config and
+    #   pool index
+    def initialize(size: 5, timeout: 5, &producer_config)
+      self.class.send(:ensure_connection_pool_gem!)
+
+      @producer_config = producer_config
+      @pool_index = 0
+      @pool_mutex = Mutex.new
+
+      @pool = ::ConnectionPool.new(size: size, timeout: timeout) do
+        producer_index = @pool_mutex.synchronize { @pool_index += 1 }
+
+        WaterDrop::Producer.new do |config|
+          if @producer_config.arity == 2
+            @producer_config.call(config, producer_index)
+          else
+            @producer_config.call(config)
+          end
+        end
+      end
+    end
+
+    # Get pool statistics
+    #
+    # @return [Hash] Pool statistics
+    def stats
+      {
+        size: @pool.size,
+        available: @pool.available
+      }
+    end
+
+    # Shutdown the connection pool
+    def shutdown
+      @pool.shutdown do |producer|
+        producer.close! if producer&.status&.active?
+      end
+    end
+
+    # Reload all connections in the pool
+    # Useful for configuration changes or error recovery
+    def reload
+      @pool.reload do |producer|
+        producer.close! if producer&.status&.active?
+      end
+    end
+
+    # Returns the underlying connection_pool instance
+    # This allows access to advanced connection_pool features if needed
+    #
+    # @return [::ConnectionPool] The underlying connection pool
+    attr_reader :pool
+  end
+
+  # Convenience methods on the WaterDrop module for global pool access
+  class << self
+    # Execute a block with a producer from the global connection pool
+    # Only available when connection pool is configured
+    #
+    # @param block [Proc] Block to execute with a producer
+    # @yield [producer] Producer from the global pool
+    # @return [Object] Result of the block
+    #
+    # @example
+    #   WaterDrop.with do |producer|
+    #     producer.produce_sync(topic: 'events', payload: 'data')
+    #   end
+    def with(&block)
+      ConnectionPool.with(&block)
+    end
+
+    # Access the global connection pool
+    #
+    # @return [WaterDrop::ConnectionPool] The global pool
+    #
+    # @example
+    #   WaterDrop.pool.with do |producer|
+    #     producer.produce_async(topic: 'events', payload: 'data')
+    #   end
+    def pool
+      ConnectionPool.default_pool
+    end
+  end
+end

data/lib/waterdrop/instrumentation/class_monitor.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Instrumentation
+    # WaterDrop class-level instrumentation monitor for global events
+    # This monitor only supports class-level lifecycle events, not per-producer events
+    class ClassMonitor < ::Karafka::Core::Monitoring::Monitor
+      # @param notifications_bus [Object] class-level notifications bus
+      # @param namespace [String, nil] namespace for events or nil if no namespace
+      def initialize(
+        notifications_bus = WaterDrop::Instrumentation::ClassNotifications.new,
+        namespace = nil
+      )
+        super(notifications_bus, namespace)
+      end
+    end
+  end
+end

data/lib/waterdrop/instrumentation/class_notifications.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Instrumentation
+    # Class-level notifications for WaterDrop global instrumentation
+    # This only supports events that occur at the class/module level, not per-producer instance
+    class ClassNotifications < ::Karafka::Core::Monitoring::Notifications
+      # List of events that are available at the class level via WaterDrop.instrumentation
+      # These are lifecycle events for producer creation and configuration
+      EVENTS = %w[
+        producer.created
+        producer.configured
+      ].freeze
+
+      # @return [WaterDrop::Instrumentation::ClassNotifications] class-level notification instance
+      def initialize
+        super
+        EVENTS.each { |event| register_event(event) }
+      end
+    end
+  end
+end

data/lib/waterdrop/instrumentation/notifications.rb

@@ -4,7 +4,7 @@ module WaterDrop
   module Instrumentation
     # Instrumented is used to hookup external monitoring services to monitor how WaterDrop works
     class Notifications < ::Karafka::Core::Monitoring::Notifications
-      # List of events that we support in the system and to which a monitor client can hook up
+      # List of events that we support at the instance level (per-producer instrumentation)
       # @note The non-error once support timestamp benchmarking
       EVENTS = %w[
         producer.connected

data/lib/waterdrop/producer/class_monitor.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  class Producer
+    # Module that provides class-level instrumentation capabilities to producers
+    # This allows producers to emit lifecycle events that can be subscribed to at the global level
+    module ClassMonitor
+      private
+
+      # @return [WaterDrop::Instrumentation::ClassMonitor] global class-level monitor for
+      #   instrumentation events
+      def class_monitor
+        WaterDrop.instrumentation
+      end
+    end
+  end
+end

data/lib/waterdrop/producer.rb

@@ -8,6 +8,7 @@ module WaterDrop
     include Async
     include Buffer
     include Transactions
+    include ClassMonitor
     include ::Karafka::Core::Helpers::Time
     include ::Karafka::Core::Taggable
 
@@ -60,6 +61,13 @@ module WaterDrop
       @status = Status.new
       @messages = []
 
+      # Instrument producer creation for global listeners
+      class_monitor.instrument(
+        'producer.created',
+        producer: self,
+        producer_id: @id
+      )
+
       return unless block
 
       setup(&block)
@@ -80,7 +88,19 @@ module WaterDrop
       @contract = Contracts::Message.new(max_payload_size: @config.max_payload_size)
       @default_variant = Variant.new(self, default: true)
 
-      return @status.configured! if @config.idle_disconnect_timeout.zero?
+      if @config.idle_disconnect_timeout.zero?
+        @status.configured!
+
+        # Instrument producer configuration for global listeners
+        class_monitor.instrument(
+          'producer.configured',
+          producer: self,
+          producer_id: @id,
+          config: @config
+        )
+
+        return
+      end
 
       # Setup idle disconnect listener if configured so we preserve tcp connections on rarely
       # used producers
@@ -92,6 +112,14 @@ module WaterDrop
       @monitor.subscribe(disconnector)
 
       @status.configured!
+
+      # Instrument producer configuration for global listeners
+      class_monitor.instrument(
+        'producer.configured',
+        producer: self,
+        producer_id: @id,
+        config: @config
+      )
     end
 
     # @return [Rdkafka::Producer] raw rdkafka producer

data/lib/waterdrop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '2.8.7'
+  VERSION = '2.8.8.rc1'
 end

data/lib/waterdrop.rb

@@ -16,6 +16,22 @@ module WaterDrop
     def gem_root
       Pathname.new(File.expand_path('..', __dir__))
     end
+
+    # @return [WaterDrop::Instrumentation::ClassMonitor] global instrumentation monitor for
+    #   class-level event subscriptions. This allows external libraries to subscribe to WaterDrop
+    #   lifecycle events without needing producer instance references.
+    #
+    # @note Only supports class-level events (producer.created, producer.configured), not
+    #   instance events
+    #
+    # @example Subscribe to producer creation events
+    #   WaterDrop.instrumentation.subscribe('producer.created') do |event|
+    #     producer = event[:producer]
+    #     # Configure producer or add middleware
+    #   end
+    def instrumentation
+      @instrumentation ||= Instrumentation::ClassMonitor.new
+    end
   end
 end
 

data/waterdrop.gemspec

@@ -27,7 +27,6 @@ Gem::Specification.new do |spec|
   spec.require_paths = %w[lib]
 
   spec.metadata = {
-    'funding_uri' => 'https://karafka.io/#become-pro',
     'homepage_uri' => 'https://karafka.io',
     'changelog_uri' => 'https://karafka.io/docs/Changelog-WaterDrop',
     'bug_tracker_uri' => 'https://github.com/karafka/waterdrop/issues',

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 2.8.7
+  version: 2.8.8.rc1
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -94,6 +94,7 @@ files:
 - lib/waterdrop/clients/dummy.rb
 - lib/waterdrop/clients/rdkafka.rb
 - lib/waterdrop/config.rb
+- lib/waterdrop/connection_pool.rb
 - lib/waterdrop/contracts.rb
 - lib/waterdrop/contracts/config.rb
 - lib/waterdrop/contracts/message.rb
@@ -105,6 +106,8 @@ files:
 - lib/waterdrop/instrumentation/callbacks/error.rb
 - lib/waterdrop/instrumentation/callbacks/oauthbearer_token_refresh.rb
 - lib/waterdrop/instrumentation/callbacks/statistics.rb
+- lib/waterdrop/instrumentation/class_monitor.rb
+- lib/waterdrop/instrumentation/class_notifications.rb
 - lib/waterdrop/instrumentation/idle_disconnector_listener.rb
 - lib/waterdrop/instrumentation/logger_listener.rb
 - lib/waterdrop/instrumentation/monitor.rb
@@ -116,6 +119,7 @@ files:
 - lib/waterdrop/producer/async.rb
 - lib/waterdrop/producer/buffer.rb
 - lib/waterdrop/producer/builder.rb
+- lib/waterdrop/producer/class_monitor.rb
 - lib/waterdrop/producer/status.rb
 - lib/waterdrop/producer/sync.rb
 - lib/waterdrop/producer/transactions.rb
@@ -129,7 +133,6 @@ licenses:
 - LGPL-3.0-only
 - Commercial
 metadata:
-  funding_uri: https://karafka.io/#become-pro
   homepage_uri: https://karafka.io
   changelog_uri: https://karafka.io/docs/Changelog-WaterDrop
   bug_tracker_uri: https://github.com/karafka/waterdrop/issues