waterdrop 2.8.5 → 2.8.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 724d3ad251e8ffee9c1fa855dd65059c3fdcedc954f3acb27bedfa226ee4d9f9
-  data.tar.gz: 43f902292f2f14a1f40de8650f1549648a821b5faae02978cb3ea179eb4e6100
+  metadata.gz: c6fd5bbc4f935b3d55794ad4f62970ab9feab3d4ebd100f9394c1bb456b8a9b5
+  data.tar.gz: 00ab23a92637285766a29741324b65422b77aba3522aa812b0f72350e3061585
 SHA512:
-  metadata.gz: 41dd3a79e7d0d6bba6c95ff6c60ba01511a118c069038a93b544c37a389b6e26868654074849fe9671f399ebb114f49537c1237f4431be463f25d9f52925e596
-  data.tar.gz: 51e22d8a542075b1e1c83c5c339b89b20f1db313614b6edbf573e59387b43bcd516d9e30e5f613b02691786ae8dc8166bc471581bc8d780eb6eafbe17353b666
+  metadata.gz: 0a41a342c8b16e167f1292ea6068647ac100f21f7f632342280a0bf69debcee3e6929f070e63b0fde2c28b85454edc6845d66895376a953c3ccaad48873646df
+  data.tar.gz: 89956ae428a553f91d91a8064954dea2095cb81457fd78f1565e31ec47e31799b823942931a08f1e6f2d0a7f04b7f730d5d1856fe01fd4de9a57055b5e301360

data/.github/workflows/ci.yml

@@ -6,9 +6,7 @@ concurrency:
 
 on:
   pull_request:
-    branches: [ main, master ]
-  push:
-    branches: [ main, master ]
+    branches: [ master ]
   schedule:
     - cron:  '0 1 * * *'
 
@@ -20,6 +18,8 @@ jobs:
     timeout-minutes: 15
     runs-on: ubuntu-latest
     needs: diffend
+    env:
+      BUNDLE_FORCE_RUBY_PLATFORM: ${{ matrix.force_ruby_platform }}
     strategy:
       fail-fast: false
       matrix:
@@ -29,6 +29,9 @@ jobs:
           - '3.3'
           - '3.2'
           - '3.1'
+        force_ruby_platform:
+          - true
+          - false
         include:
           - ruby: '3.4'
             coverage: 'true'
@@ -43,14 +46,15 @@ jobs:
       - name: Remove platform-specific ffi entries for Ruby previews
         if: contains(matrix.ruby, '3.5')
         run: |
-          ruby -i -ne 'puts $_ unless /^\s*ffi \(.*-.*\)$/' Gemfile.lock
+          sed -i '/^\s*ffi (.*-.*)$/d' Gemfile.lock
 
       - name: Set up Ruby
-        uses: ruby/setup-ruby@a4effe49ee8ee5b8b5091268c473a4628afb5651 # v1.245.0
+        uses: ruby/setup-ruby@2a7b30092b0caf9c046252510f9273b4875f3db9 # v1.254.0
         with:
           ruby-version: ${{matrix.ruby}}
           bundler-cache: true
           bundler: 'latest'
+          self-hosted: false
 
       - name: Run Kafka with Docker Compose
         run: |
@@ -75,6 +79,9 @@ jobs:
           GITHUB_COVERAGE: ${{matrix.coverage}}
         run: bundle exec rspec
 
+      - name: Run integration tests
+        run: ./bin/integrations
+
       - name: Check Kafka logs for unexpected warnings
         run: bin/verify_kafka_warnings
 
@@ -83,6 +90,7 @@ jobs:
 
   diffend:
     timeout-minutes: 5
+
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -91,9 +99,11 @@ jobs:
         with:
           fetch-depth: 0
       - name: Set up Ruby
-        uses: ruby/setup-ruby@a4effe49ee8ee5b8b5091268c473a4628afb5651 # v1.245.0
+        uses: ruby/setup-ruby@2a7b30092b0caf9c046252510f9273b4875f3db9 # v1.254.0
         with:
           ruby-version: 3.4
+          self-hosted: false
+
       - name: Install latest bundler
         run: gem install bundler --no-document
       - name: Install Diffend plugin
@@ -124,3 +134,20 @@ jobs:
           fi
       - name: Run Coditsu
         run: ./coditsu_script.sh
+
+  ci-success:
+    name: CI Success
+    runs-on: ubuntu-latest
+    if: always()
+    needs:
+      - diffend
+      - coditsu
+      - specs
+    steps:
+      - name: Check all jobs passed
+        if: |
+          contains(needs.*.result, 'failure') ||
+          contains(needs.*.result, 'cancelled') ||
+          contains(needs.*.result, 'skipped')
+        run: exit 1
+      - run: echo "All CI checks passed!"

data/.github/workflows/push.yml

@@ -24,7 +24,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Ruby
-        uses: ruby/setup-ruby@a4effe49ee8ee5b8b5091268c473a4628afb5651 # v1.245.0
+        uses: ruby/setup-ruby@2a7b30092b0caf9c046252510f9273b4875f3db9 # v1.254.0
         with:
           bundler-cache: false
 

data/.github/workflows/trigger-wiki-refresh.yml

@@ -0,0 +1,37 @@
+name: Trigger Wiki Refresh
+
+on:
+  release:
+    types: [published]
+  push:
+    branches: [master]
+
+jobs:
+  trigger-wiki-refresh:
+    runs-on: >-
+      ${{
+        (github.event_name != 'pull_request' ||
+         github.event.pull_request.head.repo.full_name == github.repository)
+        && fromJSON('["self-hosted", "linux", "x64", "qemu"]')
+        || 'ubuntu-latest'
+      }}
+
+    environment: wiki-trigger
+    if: github.repository_owner == 'karafka'
+    steps:
+      - name: Trigger wiki refresh
+        uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # v3.0.0
+        with:
+          token: ${{ secrets.WIKI_REPO_TOKEN }}
+          repository: karafka/wiki
+          event-type: sync-trigger
+          client-payload: |
+            {
+              "repository": "${{ github.repository }}",
+              "event_name": "${{ github.event_name }}",
+              "release_tag": "${{ github.event.release.tag_name || '' }}",
+              "release_name": "${{ github.event.release.name || '' }}",
+              "commit_sha": "${{ github.sha }}",
+              "commit_message": "Trigger Wiki Refresh",
+              "triggered_by": "${{ github.actor }}"
+            }

data/.github/workflows/verify-action-pins.yml

@@ -4,8 +4,15 @@ on:
     paths:
       - '.github/workflows/**'
 jobs:
-  verify:
-    runs-on: ubuntu-latest
+  verify_action_pins:
+    runs-on: >-
+      ${{
+        (github.event_name != 'pull_request' ||
+         github.event.pull_request.head.repo.full_name == github.repository)
+        && fromJSON('["self-hosted", "linux", "x64", "qemu"]')
+        || 'ubuntu-latest'
+      }}
+
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Check SHA pins

data/.rspec

@@ -1 +1,2 @@
 --require spec_helper
+--exclude-pattern "spec/integrations/**/*_spec.rb"

data/.ruby-version

@@ -1 +1 @@
-3.4.4
+3.4.5

data/CHANGELOG.md

@@ -1,5 +1,21 @@
 # WaterDrop changelog
 
+## 2.8.7 (2025-09-02)
+- [Enhancement] Disable Nagle algorithm by default (improves latency / aligned with librdkafka)
+- [Change] Normalize how libs and dependencies are required (no functional change for the end user)
+
+## 2.8.6 (2025-08-18)
+- [Feature] Add `idle_disconnect_timeout` config option to automatically disconnect idle producers after a configurable timeout period.
+- [Feature] Add support for [async](https://github.com/socketry/async) gems ecosystem with proper fiber yielding during blocking operations.
+- [Feature] Add integration testing infrastructure with `bin/integrations` runner for testing external ecosystem compatibility.
+- [Enhancement] Introduce the `WaterDrop::Producer#disconnect` so users can write custom logic to save on connections then producer is only used from time to time.
+- [Enhancement] Introduce `WaterDrop::Producer#inspect` that is mutex-safe.
+- [Enhancement] Raise errors on detected Ruby warnings.
+- [Enhancement] Optimize producer for Ruby shapes.
+- [Enhancement] Add integration spec to validate fiber yielding behavior with async gems.
+- [Change] Require `karafka-rdkafka` `>=` `0.20.0`.
+- [Change] Add new CI action to trigger auto-doc refresh.
+
 ## 2.8.5 (2025-06-23)
 - [Enhancement] Normalize topic + partition logs format (single place).
 - [Fix] A producer is not idempotent unless the enable.idempotence config is `true` (ferrous26).

data/Gemfile

@@ -17,4 +17,5 @@ group :test do
   gem 'ostruct'
   gem 'rspec'
   gem 'simplecov'
+  gem 'warning'
 end

data/Gemfile.lock

@@ -1,9 +1,9 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.8.5)
+    waterdrop (2.8.7)
       karafka-core (>= 2.4.9, < 3.0.0)
-      karafka-rdkafka (>= 0.19.2)
+      karafka-rdkafka (>= 0.20.0)
       zeitwerk (~> 2.3)
 
 GEM
@@ -23,16 +23,43 @@ GEM
     ffi (1.17.2-x86_64-darwin)
     ffi (1.17.2-x86_64-linux-gnu)
     ffi (1.17.2-x86_64-linux-musl)
-    karafka-core (2.4.11)
-      karafka-rdkafka (>= 0.17.6, < 0.20.0)
+    json (2.13.2)
+    karafka-core (2.5.6)
+      karafka-rdkafka (>= 0.20.0)
       logger (>= 1.6.0)
-    karafka-rdkafka (0.19.5)
+    karafka-rdkafka (0.21.0)
       ffi (~> 1.15)
+      json (> 2.0)
+      logger
+      mini_portile2 (~> 2.6)
+      rake (> 12)
+    karafka-rdkafka (0.21.0-aarch64-linux-gnu)
+      ffi (~> 1.15)
+      json (> 2.0)
+      logger
+      mini_portile2 (~> 2.6)
+      rake (> 12)
+    karafka-rdkafka (0.21.0-arm64-darwin)
+      ffi (~> 1.15)
+      json (> 2.0)
+      logger
+      mini_portile2 (~> 2.6)
+      rake (> 12)
+    karafka-rdkafka (0.21.0-x86_64-linux-gnu)
+      ffi (~> 1.15)
+      json (> 2.0)
+      logger
+      mini_portile2 (~> 2.6)
+      rake (> 12)
+    karafka-rdkafka (0.21.0-x86_64-linux-musl)
+      ffi (~> 1.15)
+      json (> 2.0)
+      logger
       mini_portile2 (~> 2.6)
       rake (> 12)
     logger (1.7.0)
     mini_portile2 (2.8.9)
-    ostruct (0.6.2)
+    ostruct (0.6.3)
     rake (13.3.0)
     rspec (3.13.1)
       rspec-core (~> 3.13.0)
@@ -53,6 +80,7 @@ GEM
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.13.1)
     simplecov_json_formatter (0.1.4)
+    warning (1.5.0)
     zeitwerk (2.6.18)
 
 PLATFORMS
@@ -73,8 +101,9 @@ DEPENDENCIES
   ostruct
   rspec
   simplecov
+  warning
   waterdrop!
   zeitwerk (~> 2.6.18)
 
 BUNDLED WITH
-   2.6.7
+   2.7.0

data/README.md

@@ -15,6 +15,7 @@ It:
  - Supports producing to multiple clusters
  - Supports multiple delivery policies
  - Supports per-topic configuration alterations (variants)
+ - Works with [async](https://github.com/socketry/async) gems ecosystem
  - Works with Kafka `1.0+` and Ruby `3.1+`
  - Works with and without Karafka
 

data/bin/integrations

@@ -0,0 +1,242 @@
+#!/usr/bin/env ruby
+
+# Runner to run integration specs
+
+# All integration specs run with their own bundler context to avoid dependency conflicts.
+# All WaterDrop integration specs are pristine by default since they use isolated Gemfiles.
+raise 'This code needs to be executed WITHOUT bundle exec' if Kernel.const_defined?(:Bundler)
+
+require 'open3'
+require 'fileutils'
+require 'pathname'
+require 'tmpdir'
+
+ROOT_PATH = Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../')))
+
+# How may bytes do we want to keep from the stdout in the buffer for when we need to print it
+MAX_BUFFER_OUTPUT = 307_200
+
+# Abstraction around a single test scenario execution process
+class Scenario
+  # How long a scenario can run before we kill it
+  # This is a fail-safe just in case something would hang
+  MAX_RUN_TIME = 5 * 60 # 5 minutes tops
+
+  # Expected exit codes for each integration test
+  # All WaterDrop integration tests should exit with 0 on success, 1 on failure
+  EXIT_CODES = {
+    default: [0]
+  }.freeze
+
+  private_constant :MAX_RUN_TIME, :EXIT_CODES
+
+  # Creates scenario instance and runs in the background process
+  #
+  # @param path [String] path to the scenarios file
+  def initialize(path)
+    @path = path
+    # First 1024 characters from stdout
+    @stdout_head = ''
+    # Last 1024 characters from stdout
+    @stdout_tail = ''
+  end
+
+  # Starts running given scenario in a separate process
+  def start
+    @stdin, @stdout, @stderr, @wait_thr = Open3.popen3(init_and_build_cmd)
+    @started_at = current_time
+  end
+
+  # @return [String] integration spec name
+  def name
+    @path.gsub("#{ROOT_PATH}/spec/integrations/", '')
+  end
+
+
+  # @return [Boolean] did this scenario finished or is it still running
+  def finished?
+    # If the thread is running too long, kill it
+    if current_time - @started_at > MAX_RUN_TIME
+      begin
+        Process.kill('TERM', pid)
+      # It may finish right after we want to kill it, that's why we ignore this
+      rescue Errno::ESRCH
+      end
+    end
+
+    # We read it so it won't grow as we use our default logger that prints to both test.log and
+    # to stdout. Otherwise after reaching the buffer size, it would hang
+    buffer = ''
+    @stdout.read_nonblock(MAX_BUFFER_OUTPUT, buffer, exception: false)
+    @stdout_head = buffer if @stdout_head.empty?
+    @stdout_tail << buffer
+    @stdout_tail = @stdout_tail[-MAX_BUFFER_OUTPUT..-1] || @stdout_tail
+
+    !@wait_thr.alive?
+  end
+
+  # @return [Boolean] did this scenario finish successfully or not
+  def success?
+    expected_exit_codes = EXIT_CODES[name] || EXIT_CODES[:default]
+
+    expected_exit_codes.include?(exit_code)
+  end
+
+  # @return [Integer] pid of the process of this scenario
+  def pid
+    @wait_thr.pid
+  end
+
+  # @return [Integer] exit code of the process running given scenario
+  def exit_code
+    # There may be no exit status if we killed the thread
+    @wait_thr.value&.exitstatus || 123
+  end
+
+  # @return [String] exit status of the process
+  def exit_status
+    @wait_thr.value.to_s
+  end
+
+  # Prints a status report when scenario is finished and stdout if it failed
+  def report
+    if success?
+      print "\e[#{32}m#{'.'}\e[0m"
+    else
+      buffer = ''
+
+      @stderr.read_nonblock(MAX_BUFFER_OUTPUT, buffer, exception: false)
+
+      puts
+      puts "\e[#{31}m#{'[FAILED]'}\e[0m #{name}"
+      puts "Time taken: #{current_time - @started_at} seconds"
+      puts "Exit code: #{exit_code}"
+      puts "Exit status: #{exit_status}"
+      puts @stdout_head
+      puts '...'
+      puts @stdout_tail
+      puts buffer
+      puts
+    end
+  end
+
+  # @return [Float] number of seconds that a given spec took to run
+  def time_taken
+    @finished_at - @started_at
+  end
+
+  # Close all the files that are open, so they do not pile up
+  def close
+    @finished_at = current_time
+    @stdin.close
+    @stdout.close
+    @stderr.close
+  end
+
+  private
+
+  # Sets up a proper environment for a given spec to run and returns the run command
+  # All WaterDrop integration specs run in pristine mode with isolated Gemfiles
+  # @return [String] run command
+  def init_and_build_cmd
+    scenario_dir = File.dirname(@path)
+    # We copy the spec into a temp dir, not to pollute the spec location with logs, etc
+    temp_dir = Dir.mktmpdir
+    file_name = File.basename(@path)
+
+    FileUtils.cp_r("#{scenario_dir}/.", temp_dir)
+
+    <<~CMD
+      cd #{temp_dir} &&
+      WATERDROP_GEM_DIR=#{ROOT_PATH} \
+      bundle install &&
+      BUNDLE_AUTO_INSTALL=true \
+      WATERDROP_GEM_DIR=#{ROOT_PATH} \
+      bundle exec ruby #{file_name}
+    CMD
+  end
+
+  # @return [Float] current machine time
+  def current_time
+    Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  end
+end
+
+# Load all the specs
+specs = Dir[ROOT_PATH.join('spec/integrations/**/*_spec.rb')]
+
+FILTER_TYPE = ARGV[0] == '--exclude' ? 'exclude' : 'include'
+
+# Remove the exclude flag
+ARGV.shift if FILTER_TYPE == '--exclude'
+
+# If filters is provided, apply
+# Allows to provide several filters one after another and applies all of them
+ARGV.each do |filter|
+  specs.delete_if do |name|
+    case FILTER_TYPE
+    when 'include'
+      !name.include?(filter)
+    when 'exclude'
+      name.include?(filter)
+    else
+      raise 'Invalid filter type'
+    end
+  end
+end
+
+# Randomize order
+seed = (ENV['SPECS_SEED'] || rand(0..10_000)).to_i
+
+puts "Random seed: #{seed}"
+
+scenarios = specs
+            .shuffle(random: Random.new(seed))
+            .map { |integration| Scenario.new(integration) }
+
+raise ArgumentError, "No integration specs with filters: #{ARGV.join(', ')}" if scenarios.empty?
+
+puts "Running #{scenarios.size} scenarios"
+
+finished_scenarios = []
+
+scenarios.each do |scenario|
+  scenario.start
+
+  # Wait for this scenario to finish before moving to the next one
+  until scenario.finished?
+    sleep(0.1)
+  end
+
+  scenario.report
+  scenario.close
+  finished_scenarios << scenario
+end
+
+# Report longest scenarios
+puts
+puts "\nLongest scenarios:\n\n"
+
+finished_scenarios.sort_by(&:time_taken).reverse.first(10).each do |long_scenario|
+  puts "[#{'%6.2f' % long_scenario.time_taken}] #{long_scenario.name}"
+end
+
+failed_scenarios = finished_scenarios.reject(&:success?)
+
+if failed_scenarios.empty?
+  puts
+else
+  # Report once more on the failed jobs
+  # This will only list scenarios that failed without printing their stdout here.
+  puts
+  puts "\nFailed scenarios:\n\n"
+
+  failed_scenarios.each do |scenario|
+    puts "\e[#{31}m#{'[FAILED]'}\e[0m #{scenario.name}"
+  end
+
+  puts
+
+  # Exit with 1 if not all scenarios were successful
+  exit 1
+end

data/bin/verify_kafka_warnings

@@ -9,7 +9,7 @@ allowed_patterns=(
 )
 
 # Get all warnings
-warnings=$(docker logs --since=0 kafka | grep WARN)
+warnings=$(docker logs --since=0 kafka | grep "] WARN ")
 exit_code=0
 
 while IFS= read -r line; do

data/config/locales/errors.yml

@@ -19,6 +19,7 @@ en:
       max_attempts_on_transaction_command_format: must be an integer that is equal or bigger than 1
       reload_on_transaction_fatal_error_format: must be boolean
       oauth.token_provider_listener_format: 'must be false or respond to #on_oauthbearer_token_refresh'
+      idle_disconnect_timeout_format: 'must be an integer that is equal to 0 or bigger than 30 000 (30 seconds)'
 
     variant:
       missing: must be present

data/lib/waterdrop/config.rb

@@ -17,7 +17,10 @@ module WaterDrop
       # sync delivery
       'message.timeout.ms': 50_000,
       # Must be more or equal to `message.timeout.ms` defaults
-      'transaction.timeout.ms': 55_000
+      'transaction.timeout.ms': 55_000,
+      # Lowers latency. Default in newer librdkafka but we want to make sure it is shipped to
+      # users despite what librdkafka they run on
+      'socket.nagle.disable': true
     }.freeze
 
     private_constant :KAFKA_DEFAULTS
@@ -76,6 +79,12 @@ module WaterDrop
     # to keep going or should we stop. Since we will open a new instance and the failed transaction
     # anyhow rolls back, we should be able to safely reload.
     setting :reload_on_transaction_fatal_error, default: true
+    # option [Integer] Idle disconnect timeout in milliseconds. When set to 0, idle disconnection
+    #   is disabled. When set to a positive value, WaterDrop will automatically disconnect
+    #   producers that haven't sent any messages for the specified time period. This helps preserve
+    #   TCP connections in low-intensity scenarios. Minimum value is 30 seconds (30_000 ms) to
+    #   prevent overly aggressive disconnections.
+    setting :idle_disconnect_timeout, default: 0
 
     # option [Boolean] should we send messages. Setting this to false can be really useful when
     #   testing and or developing because when set to false, won't actually ping Kafka but will

data/lib/waterdrop/contracts/config.rb

@@ -27,6 +27,9 @@ module WaterDrop
       required(:wait_backoff_on_transaction_command) { |val| val.is_a?(Numeric) && val >= 0 }
       required(:max_attempts_on_transaction_command) { |val| val.is_a?(Integer) && val >= 1 }
       required(:reload_on_transaction_fatal_error) { |val| [true, false].include?(val) }
+      required(:idle_disconnect_timeout) do |val|
+        val.is_a?(Integer) && (val.zero? || val >= 30_000)
+      end
 
       nested(:oauth) do
         required(:token_provider_listener) do |val|

data/lib/waterdrop/instrumentation/idle_disconnector_listener.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Instrumentation
+    # Idle disconnector listener that monitors producer activity and automatically disconnects
+    # idle producers to preserve TCP connections
+    #
+    # This listener subscribes to statistics.emitted events and tracks the txmsgs (transmitted
+    # messages) count. If the producer doesn't send any messages for a configurable timeout
+    # period, it will automatically disconnect the producer.
+    #
+    # @note We do not have to worry about the running transactions or buffer being used because
+    #   the disconnect is graceful and will not disconnect unless it is allowed to. This is why
+    #   we can simplify things and take interest only in txmsgs.
+    #
+    # @note For convenience, WaterDrop provides a config shortcut. Instead of manually subscribing
+    #   this listener, you can simply set `config.idle_disconnect_timeout` in your producer config.
+    #
+    # @example Using config shortcut (recommended)
+    #   WaterDrop::Producer.new do |config|
+    #     config.idle_disconnect_timeout = 5 * 60 * 1000 # 5 minutes
+    #   end
+    #
+    # @example Manual listener usage with 5 minute timeout
+    #   producer.monitor.subscribe(
+    #     WaterDrop::Instrumentation::IdleDisconnectorListener.new(
+    #       producer,
+    #       disconnect_timeout: 5 * 60 * 1000)
+    #   )
+    #
+    # @example Usage with custom timeout
+    #   idle_disconnector = WaterDrop::Instrumentation::IdleDisconnectorListener.new(
+    #     producer,
+    #     disconnect_timeout: 10 * 60 * 1000
+    #   )
+    #   producer.monitor.subscribe(idle_disconnector)
+    class IdleDisconnectorListener
+      include ::Karafka::Core::Helpers::Time
+
+      # @param producer [WaterDrop::Producer] the producer instance to monitor
+      # @param disconnect_timeout [Integer] timeout in milliseconds before disconnecting
+      #   (default: 5 minutes). Be aware that if you set it to a value lower than statistics
+      #   publishing interval (5 seconds by default) it may be to aggressive in closing
+      def initialize(producer, disconnect_timeout: 5 * 60 * 1_000)
+        @producer = producer
+        @disconnect_timeout = disconnect_timeout
+        # We set this initially to -1 so any statistics change triggers a change to prevent an
+        # early shutdown
+        @last_txmsgs = -1
+        @last_activity_time = monotonic_now
+      end
+
+      # This method is called automatically when the listener is subscribed to the monitor
+      # using producer.monitor.subscribe(listener_instance)
+      #
+      # @param event [Hash] the statistics event containing producer statistics
+      def on_statistics_emitted(event)
+        call(event[:statistics])
+      end
+
+      private
+
+      # Handles statistics.emitted events to monitor message transmission activity
+      # @param statistics [Hash] producer librdkafka statistics
+      def call(statistics)
+        current_txmsgs = statistics.fetch('txmsgs', 0)
+        current_time = monotonic_now
+
+        # Update activity if messages changed
+        if current_txmsgs != @last_txmsgs
+          @last_txmsgs = current_txmsgs
+          @last_activity_time = current_time
+
+          return
+        end
+
+        # Check for timeout and attempt disconnect
+        return unless (current_time - @last_activity_time) >= @disconnect_timeout
+
+        if @producer.disconnectable?
+          # Since the statistics operations happen from the rdkafka native thread. we cannot close
+          # it from itself as you cannot join on yourself as it would cause a deadlock. We spawn
+          # a thread to do this
+          # We do an early check if producer is in a viable state for a disconnect so in case its
+          # internal state would prevent us from disconnecting, we won't be spamming with new
+          # thread creation
+          Thread.new do
+            @producer.disconnect
+          rescue StandardError => e
+            @producer.monitor.instrument(
+              'error.occurred',
+              producer_id: @producer.id,
+              error: e,
+              type: 'producer.disconnect.error'
+            )
+          end
+        end
+
+        # We change this always because:
+        #   - if we were able to disconnect, this should give us time before any potential future
+        #     attempts. While they should not happen because events won't be published on a
+        #     disconnected producer, this may still with frequent events be called post disconnect
+        #   - if we were not able to disconnect, it means that there was something in the producer
+        #     state that prevent it, and we consider this as activity as well
+        @last_activity_time = current_time
+      end
+    end
+  end
+end

data/lib/waterdrop/instrumentation/logger_listener.rb

@@ -132,6 +132,18 @@ module WaterDrop
         info(event, 'Closing producer')
       end
 
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_producer_disconnecting(event)
+        info(event, 'Disconnecting producer')
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      # @note While this says "Disconnecting producer", it produces a nice message with time taken:
+      #   "Disconnecting producer took 5 ms" indicating it happened in the past.
+      def on_producer_disconnected(event)
+        info(event, 'Disconnected producer')
+      end
+
       # @param event [Dry::Events::Event] event that happened with the details
       def on_producer_reloaded(event)
         info(event, 'Producer successfully reloaded')

data/lib/waterdrop/instrumentation/notifications.rb

@@ -11,6 +11,8 @@ module WaterDrop
         producer.closing
         producer.closed
         producer.reloaded
+        producer.disconnecting
+        producer.disconnected
 
         message.produced_async
         message.produced_sync

data/lib/waterdrop/producer/status.rb

@@ -9,6 +9,8 @@ module WaterDrop
         initial
         configured
         connected
+        disconnecting
+        disconnected
         closing
         closed
       ].freeze
@@ -22,11 +24,12 @@ module WaterDrop
       end
 
       # @return [Boolean] true if producer is in a active state. Active means, that we can start
-      #   sending messages. Actives states are connected (connection established) or configured,
-      #   which means, that producer is configured, but connection with Kafka is
-      #   not yet established.
+      #   sending messages. Active states are connected (connection established), configured
+      #   which means, that producer is configured, but connection with Kafka is not yet
+      #   established or disconnected, meaning it was working but user disconnected for his own
+      #   reasons though sending could reconnect and continue.
       def active?
-        connected? || configured?
+        connected? || configured? || disconnecting? || disconnected?
       end
 
       # @return [String] current status as a string

data/lib/waterdrop/producer.rb

@@ -50,6 +50,12 @@ module WaterDrop
       @connecting_mutex = Mutex.new
       @operating_mutex = Mutex.new
       @transaction_mutex = Mutex.new
+      @id = nil
+      @monitor = nil
+      @contract = nil
+      @default_variant = nil
+      @client = nil
+      @closing_thread_id = nil
 
       @status = Status.new
       @messages = []
@@ -73,6 +79,18 @@ module WaterDrop
       @monitor = @config.monitor
       @contract = Contracts::Message.new(max_payload_size: @config.max_payload_size)
       @default_variant = Variant.new(self, default: true)
+
+      return @status.configured! if @config.idle_disconnect_timeout.zero?
+
+      # Setup idle disconnect listener if configured so we preserve tcp connections on rarely
+      # used producers
+      disconnector = Instrumentation::IdleDisconnectorListener.new(
+        self,
+        disconnect_timeout: @config.idle_disconnect_timeout
+      )
+
+      @monitor.subscribe(disconnector)
+
       @status.configured!
     end
 
@@ -178,6 +196,74 @@ module WaterDrop
       @middleware ||= config.middleware
     end
 
+    # Disconnects the producer from Kafka while keeping it configured for potential reconnection
+    #
+    # This method safely disconnects the underlying Kafka client while preserving the producer's
+    # configuration. Unlike `#close`, this allows the producer to be reconnected later by calling
+    # methods that require the client. The disconnection will only proceed if certain safety
+    # conditions are met.
+    #
+    # This API can be used to preserve connections on low-intensity producer instances, etc.
+    #
+    # @return [Boolean] true if disconnection was successful, false if disconnection was not
+    #   possible due to safety conditions (active transactions, ongoing operations, pending
+    #   messages in buffer, or if already disconnected)
+    #
+    # @note This method will refuse to disconnect if:
+    #   - There are pending messages in the internal buffer
+    #   - There are operations currently in progress
+    #   - A transaction is currently active
+    #   - The client is not currently connected
+    #   - Required mutexes are locked by other operations
+    #
+    # @note After successful disconnection, the producer status changes to disconnected but
+    #   remains configured, allowing for future reconnection when client access is needed.
+    def disconnect
+      return false unless disconnectable?
+
+      # Use the same mutex pattern as the regular close method to prevent race conditions
+      @transaction_mutex.synchronize do
+        @operating_mutex.synchronize do
+          @buffer_mutex.synchronize do
+            return false unless @client
+            return false unless @status.connected?
+            return false unless @messages.empty?
+            return false unless @operations_in_progress.value.zero?
+
+            @status.disconnecting!
+            @monitor.instrument('producer.disconnecting', producer_id: id)
+
+            @monitor.instrument('producer.disconnected', producer_id: id) do
+              # Close the client
+              @client.close
+              @client = nil
+
+              # Reset connection status but keep producer configured
+              @status.disconnected!
+            end
+
+            true
+          end
+        end
+      end
+    end
+
+    # Is the producer in a state from which we can disconnect
+    #
+    # @return [Boolean] is producer in a state that potentially allows for a disconnect
+    #
+    # @note This is a best effort method. The proper checks happen also when disconnecting behind
+    #   all the needed mutexes
+    def disconnectable?
+      return false unless @client
+      return false unless @status.connected?
+      return false unless @messages.empty?
+      return false if @transaction_mutex.locked?
+      return false if @operating_mutex.locked?
+
+      true
+    end
+
     # Flushes the buffers in a sync way and closes the producer
     # @param force [Boolean] should we force closing even with outstanding messages after the
     #   max wait timeout
@@ -260,6 +346,37 @@ module WaterDrop
       close(force: true)
     end
 
+    # @return [String] mutex-safe inspect details
+    def inspect
+      # Basic info that's always safe to access
+      parts = []
+      parts << "id=#{@id.inspect}"
+      parts << "status=#{@status}" if @status
+
+      # Try to get buffer info safely
+      if @buffer_mutex.try_lock
+        begin
+          parts << "buffer_size=#{@messages.size}"
+        ensure
+          @buffer_mutex.unlock
+        end
+      else
+        parts << 'buffer_size=busy'
+      end
+
+      # Check if client is connected without triggering connection
+      parts << if @status.connected?
+                 'connected=true'
+               else
+                 'connected=false'
+               end
+
+      parts << "operations=#{@operations_in_progress.value}"
+      parts << 'in_transaction=true' if @transaction_mutex.locked?
+
+      "#<#{self.class.name}:#{format('%#x', object_id)} #{parts.join(' ')}>"
+    end
+
     private
 
     # Ensures that we don't run any operations when the producer is not configured or when it

data/lib/waterdrop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '2.8.5'
+  VERSION = '2.8.7'
 end

data/lib/waterdrop.rb

@@ -1,16 +1,13 @@
 # frozen_string_literal: true
 
 # External components
-# delegate should be removed because we don't need it, we just add it because of ruby-kafka
-%w[
-  delegate
-  forwardable
-  json
-  zeitwerk
-  securerandom
-  karafka-core
-  pathname
-].each { |lib| require lib }
+require 'delegate'
+require 'forwardable'
+require 'json'
+require 'zeitwerk'
+require 'securerandom'
+require 'karafka-core'
+require 'pathname'
 
 # WaterDrop library
 module WaterDrop

data/renovate.json

@@ -7,12 +7,24 @@
     "enabled": true,
     "pinDigests": true
   },
+  "includePaths": [
+    "Gemfile",
+    "waterdrop.gemspec",
+    "spec/integrations/**/Gemfile"
+  ],
   "packageRules": [
     {
       "matchManagers": [
         "github-actions"
       ],
       "minimumReleaseAge": "7 days"
+    },
+    {
+      "matchFileNames": [
+        "spec/integrations/**/Gemfile"
+      ],
+      "groupName": "integration test dependencies",
+      "commitMessageTopic": "integration test dependencies"
     }
   ]
 }

data/waterdrop.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
   spec.licenses      = %w[LGPL-3.0-only Commercial]
 
   spec.add_dependency 'karafka-core', '>= 2.4.9', '< 3.0.0'
-  spec.add_dependency 'karafka-rdkafka', '>= 0.19.2'
+  spec.add_dependency 'karafka-rdkafka', '>= 0.20.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   spec.required_ruby_version = '>= 3.1.0'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 2.8.5
+  version: 2.8.7
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,14 +35,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.19.2
+        version: 0.20.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.19.2
+        version: 0.20.0
 - !ruby/object:Gem::Dependency
   name: zeitwerk
   requirement: !ruby/object:Gem::Requirement
@@ -72,6 +72,7 @@ files:
 - ".github/ISSUE_TEMPLATE/feature_request.md"
 - ".github/workflows/ci.yml"
 - ".github/workflows/push.yml"
+- ".github/workflows/trigger-wiki-refresh.yml"
 - ".github/workflows/verify-action-pins.yml"
 - ".gitignore"
 - ".rspec"
@@ -83,6 +84,7 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- bin/integrations
 - bin/verify_kafka_warnings
 - bin/verify_topics_naming
 - config/locales/errors.yml
@@ -103,6 +105,7 @@ files:
 - lib/waterdrop/instrumentation/callbacks/error.rb
 - lib/waterdrop/instrumentation/callbacks/oauthbearer_token_refresh.rb
 - lib/waterdrop/instrumentation/callbacks/statistics.rb
+- lib/waterdrop/instrumentation/idle_disconnector_listener.rb
 - lib/waterdrop/instrumentation/logger_listener.rb
 - lib/waterdrop/instrumentation/monitor.rb
 - lib/waterdrop/instrumentation/notifications.rb
@@ -147,7 +150,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.7
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Kafka messaging made easy!
 test_files: []