waterdrop 2.8.15 → 2.9.0

data/Gemfile.lock

@@ -1,58 +1,26 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.8.15)
-      karafka-core (>= 2.4.9, < 3.0.0)
-      karafka-rdkafka (>= 0.23.1)
+    waterdrop (2.9.0)
+      karafka-core (>= 2.5.12, < 3.0.0)
+      karafka-rdkafka (>= 0.24.0)
       zeitwerk (~> 2.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    byebug (12.0.0)
-    connection_pool (2.5.4)
-    diff-lcs (1.6.2)
+    byebug (13.0.0)
+      reline (>= 0.6.0)
+    connection_pool (3.0.2)
     docile (1.4.1)
-    ffi (1.17.2)
-    ffi (1.17.2-aarch64-linux-gnu)
-    ffi (1.17.2-aarch64-linux-musl)
-    ffi (1.17.2-arm-linux-gnu)
-    ffi (1.17.2-arm-linux-musl)
-    ffi (1.17.2-arm64-darwin)
-    ffi (1.17.2-x86-linux-gnu)
-    ffi (1.17.2-x86-linux-musl)
-    ffi (1.17.2-x86_64-darwin)
-    ffi (1.17.2-x86_64-linux-gnu)
-    ffi (1.17.2-x86_64-linux-musl)
-    json (2.15.1)
-    karafka-core (2.5.7)
+    drb (2.2.3)
+    ffi (1.17.4)
+    io-console (0.8.2)
+    json (2.19.3)
+    karafka-core (2.5.12)
       karafka-rdkafka (>= 0.20.0)
       logger (>= 1.6.0)
-    karafka-rdkafka (0.23.1)
-      ffi (~> 1.17.1)
-      json (> 2.0)
-      logger
-      mini_portile2 (~> 2.6)
-      rake (> 12)
-    karafka-rdkafka (0.23.1-aarch64-linux-gnu)
-      ffi (~> 1.17.1)
-      json (> 2.0)
-      logger
-      mini_portile2 (~> 2.6)
-      rake (> 12)
-    karafka-rdkafka (0.23.1-arm64-darwin)
-      ffi (~> 1.17.1)
-      json (> 2.0)
-      logger
-      mini_portile2 (~> 2.6)
-      rake (> 12)
-    karafka-rdkafka (0.23.1-x86_64-linux-gnu)
-      ffi (~> 1.17.1)
-      json (> 2.0)
-      logger
-      mini_portile2 (~> 2.6)
-      rake (> 12)
-    karafka-rdkafka (0.23.1-x86_64-linux-musl)
+    karafka-rdkafka (0.25.0)
       ffi (~> 1.17.1)
       json (> 2.0)
       logger
@@ -60,21 +28,17 @@ GEM
       rake (> 12)
     logger (1.7.0)
     mini_portile2 (2.8.9)
+    minitest (6.0.2)
+      drb (~> 2.0)
+      prism (~> 1.5)
+    mocha (3.1.0)
+      ruby2_keywords (>= 0.0.5)
     ostruct (0.6.3)
-    rake (13.3.0)
-    rspec (3.13.2)
-      rspec-core (~> 3.13.0)
-      rspec-expectations (~> 3.13.0)
-      rspec-mocks (~> 3.13.0)
-    rspec-core (3.13.6)
-      rspec-support (~> 3.13.0)
-    rspec-expectations (3.13.5)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.6)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.13.0)
-    rspec-support (3.13.6)
+    prism (1.9.0)
+    rake (13.3.1)
+    reline (0.6.3)
+      io-console (~> 0.5)
+    ruby2_keywords (0.0.5)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
@@ -82,35 +46,47 @@ GEM
     simplecov-html (0.13.2)
     simplecov_json_formatter (0.1.4)
     warning (1.5.0)
-    yard (0.9.37)
-    yard-lint (1.2.3)
-      yard (~> 0.9)
-      zeitwerk (~> 2.6)
-    zeitwerk (2.7.3)
+    zeitwerk (2.7.5)
 
 PLATFORMS
-  aarch64-linux-gnu
-  aarch64-linux-musl
-  arm-linux-gnu
-  arm-linux-musl
-  arm64-darwin
   ruby
-  x86-linux-gnu
-  x86-linux-musl
-  x86_64-darwin
-  x86_64-linux-gnu
-  x86_64-linux-musl
 
 DEPENDENCIES
   byebug
   connection_pool
+  minitest
+  mocha
   ostruct
-  rspec
   simplecov
   warning
   waterdrop!
-  yard-lint
   zeitwerk (~> 2.7.0)
 
+CHECKSUMS
+  byebug (13.0.0) sha256=d2263efe751941ca520fa29744b71972d39cbc41839496706f5d9b22e92ae05d
+  connection_pool (3.0.2) sha256=33fff5ba71a12d2aa26cb72b1db8bba2a1a01823559fb01d29eb74c286e62e0a
+  docile (1.4.1) sha256=96159be799bfa73cdb721b840e9802126e4e03dfc26863db73647204c727f21e
+  drb (2.2.3) sha256=0b00d6fdb50995fe4a45dea13663493c841112e4068656854646f418fda13373
+  ffi (1.17.4) sha256=bcd1642e06f0d16fc9e09ac6d49c3a7298b9789bcb58127302f934e437d60acf
+  io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc
+  json (2.19.3) sha256=289b0bb53052a1fa8c34ab33cc750b659ba14a5c45f3fcf4b18762dc67c78646
+  karafka-core (2.5.12) sha256=57cbb45a187fbe3df9b9a57af59dda7211f9969524b2afbb83792a64705860e1
+  karafka-rdkafka (0.25.0) sha256=67b316b942cf9ff7e9d7bbf9029e6f2d91eba97b4c9dc93b9f49fd207dfb80f8
+  logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203
+  mini_portile2 (2.8.9) sha256=0cd7c7f824e010c072e33f68bc02d85a00aeb6fce05bb4819c03dfd3c140c289
+  minitest (6.0.2) sha256=db6e57956f6ecc6134683b4c87467d6dd792323c7f0eea7b93f66bd284adbc3d
+  mocha (3.1.0) sha256=75f42d69ebfb1f10b32489dff8f8431d37a418120ecdfc07afe3bc183d4e1d56
+  ostruct (0.6.3) sha256=95a2ed4a4bd1d190784e666b47b2d3f078e4a9efda2fccf18f84ddc6538ed912
+  prism (1.9.0) sha256=7b530c6a9f92c24300014919c9dcbc055bf4cdf51ec30aed099b06cd6674ef85
+  rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c
+  reline (0.6.3) sha256=1198b04973565b36ec0f11542ab3f5cfeeec34823f4e54cebde90968092b1835
+  ruby2_keywords (0.0.5) sha256=ffd13740c573b7301cf7a2e61fc857b2a8e3d3aff32545d6f8300d8bae10e3ef
+  simplecov (0.22.0) sha256=fe2622c7834ff23b98066bb0a854284b2729a569ac659f82621fc22ef36213a5
+  simplecov-html (0.13.2) sha256=bd0b8e54e7c2d7685927e8d6286466359b6f16b18cb0df47b508e8d73c777246
+  simplecov_json_formatter (0.1.4) sha256=529418fbe8de1713ac2b2d612aa3daa56d316975d307244399fa4838c601b428
+  warning (1.5.0) sha256=0f12c49fea0c06757778eefdcc7771e4fd99308901e3d55c504d87afdd718c53
+  waterdrop (2.9.0)
+  zeitwerk (2.7.5) sha256=d8da92128c09ea6ec62c949011b00ed4a20242b255293dd66bf41545398f73dd
+
 BUNDLED WITH
-   2.7.0
+  4.0.6

data/README.md

@@ -16,7 +16,7 @@ It:
  - Supports multiple delivery policies
  - Supports per-topic configuration alterations (variants)
  - Works with [async](https://github.com/socketry/async) gems ecosystem
- - Works with Kafka `1.0+` and Ruby `3.1+`
+ - Works with Kafka `1.0+` and Ruby `3.2+`
  - Works with and without Karafka
 
 ## Documentation

data/Rakefile

@@ -1,4 +1,16 @@
 # frozen_string_literal: true
 
-require 'bundler/setup'
-require 'bundler/gem_tasks'
+require "bundler/setup"
+require "bundler/gem_tasks"
+require "minitest/test_task"
+
+Minitest::TestTask.create(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_globs = ["test/**/*_test.rb"]
+  # Load test_helper before minitest/autorun so SimpleCov's at_exit hook is
+  # registered first and thus runs last (LIFO), collecting coverage AFTER tests
+  t.test_prelude = 'require "test_helper"; require "minitest/autorun"'
+end
+
+task default: :test

data/bin/integrations

@@ -4,14 +4,14 @@
 
 # All integration specs run with their own bundler context to avoid dependency conflicts.
 # All WaterDrop integration specs are pristine by default since they use isolated Gemfiles.
-raise 'This code needs to be executed WITHOUT bundle exec' if Kernel.const_defined?(:Bundler)
+raise "This code needs to be executed WITHOUT bundle exec" if Kernel.const_defined?(:Bundler)
 
-require 'open3'
-require 'fileutils'
-require 'pathname'
-require 'tmpdir'
+require "open3"
+require "fileutils"
+require "pathname"
+require "tmpdir"
 
-ROOT_PATH = Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../')))
+ROOT_PATH = Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), "../")))
 
 # How may bytes do we want to keep from the stdout in the buffer for when we need to print it
 MAX_BUFFER_OUTPUT = 307_200
@@ -36,9 +36,9 @@ class Scenario
   def initialize(path)
     @path = path
     # First 1024 characters from stdout
-    @stdout_head = ''
+    @stdout_head = ""
     # Last 1024 characters from stdout
-    @stdout_tail = ''
+    @stdout_tail = ""
   end
 
   # Starts running given scenario in a separate process
@@ -49,16 +49,15 @@ class Scenario
 
   # @return [String] integration spec name
   def name
-    @path.gsub("#{ROOT_PATH}/spec/integrations/", '')
+    @path.gsub("#{ROOT_PATH}/test/integrations/", "")
   end
 
-
   # @return [Boolean] did this scenario finished or is it still running
   def finished?
     # If the thread is running too long, kill it
     if current_time - @started_at > MAX_RUN_TIME
       begin
-        Process.kill('TERM', pid)
+        Process.kill("TERM", pid)
       # It may finish right after we want to kill it, that's why we ignore this
       rescue Errno::ESRCH
       end
@@ -66,7 +65,7 @@ class Scenario
 
     # We read it so it won't grow as we use our default logger that prints to both test.log and
     # to stdout. Otherwise after reaching the buffer size, it would hang
-    buffer = ''
+    buffer = ""
     @stdout.read_nonblock(MAX_BUFFER_OUTPUT, buffer, exception: false)
     @stdout_head = buffer if @stdout_head.empty?
     @stdout_tail << buffer
@@ -101,19 +100,19 @@ class Scenario
   # Prints a status report when scenario is finished and stdout if it failed
   def report
     if success?
-      print "\e[#{32}m#{'.'}\e[0m"
+      print "\e[32m.\e[0m"
     else
-      buffer = ''
+      buffer = ""
 
       @stderr.read_nonblock(MAX_BUFFER_OUTPUT, buffer, exception: false)
 
       puts
-      puts "\e[#{31}m#{'[FAILED]'}\e[0m #{name}"
+      puts "\e[31m[FAILED]\e[0m #{name}"
       puts "Time taken: #{current_time - @started_at} seconds"
       puts "Exit code: #{exit_code}"
       puts "Exit status: #{exit_status}"
       puts @stdout_head
-      puts '...'
+      puts "..."
       puts @stdout_tail
       puts buffer
       puts
@@ -143,6 +142,7 @@ class Scenario
     # We copy the spec into a temp dir, not to pollute the spec location with logs, etc
     temp_dir = Dir.mktmpdir
     file_name = File.basename(@path)
+    spec_file_path = @path.sub("#{ROOT_PATH}/", "")
 
     FileUtils.cp_r("#{scenario_dir}/.", temp_dir)
 
@@ -152,7 +152,8 @@ class Scenario
       bundle install &&
       BUNDLE_AUTO_INSTALL=true \
       WATERDROP_GEM_DIR=#{ROOT_PATH} \
-      bundle exec ruby #{file_name}
+      SPEC_FILE_PATH=#{spec_file_path} \
+      bundle exec ruby -r "#{ROOT_PATH}/test/support/spec_hash" #{file_name}
     CMD
   end
 
@@ -163,38 +164,38 @@ class Scenario
 end
 
 # Load all the specs
-specs = Dir[ROOT_PATH.join('spec/integrations/**/*_spec.rb')]
+specs = Dir[ROOT_PATH.join("test/integrations/**/*_spec.rb")]
 
-FILTER_TYPE = ARGV[0] == '--exclude' ? 'exclude' : 'include'
+FILTER_TYPE = (ARGV[0] == "--exclude") ? "exclude" : "include"
 
 # Remove the exclude flag
-ARGV.shift if FILTER_TYPE == '--exclude'
+ARGV.shift if FILTER_TYPE == "--exclude"
 
 # If filters is provided, apply
 # Allows to provide several filters one after another and applies all of them
 ARGV.each do |filter|
   specs.delete_if do |name|
     case FILTER_TYPE
-    when 'include'
+    when "include"
       !name.include?(filter)
-    when 'exclude'
+    when "exclude"
       name.include?(filter)
     else
-      raise 'Invalid filter type'
+      raise "Invalid filter type"
     end
   end
 end
 
 # Randomize order
-seed = (ENV['SPECS_SEED'] || rand(0..10_000)).to_i
+seed = (ENV["SPECS_SEED"] || rand(0..10_000)).to_i
 
 puts "Random seed: #{seed}"
 
 scenarios = specs
-            .shuffle(random: Random.new(seed))
-            .map { |integration| Scenario.new(integration) }
+  .shuffle(random: Random.new(seed))
+  .map { |integration| Scenario.new(integration) }
 
-raise ArgumentError, "No integration specs with filters: #{ARGV.join(', ')}" if scenarios.empty?
+raise ArgumentError, "No integration specs with filters: #{ARGV.join(", ")}" if scenarios.empty?
 
 puts "Running #{scenarios.size} scenarios"
 
@@ -217,8 +218,8 @@ end
 puts
 puts "\nLongest scenarios:\n\n"
 
-finished_scenarios.sort_by(&:time_taken).reverse.first(10).each do |long_scenario|
-  puts "[#{'%6.2f' % long_scenario.time_taken}] #{long_scenario.name}"
+finished_scenarios.sort_by(&:time_taken).last(10).reverse_each do |long_scenario|
+  puts "[#{"%6.2f" % long_scenario.time_taken}] #{long_scenario.name}"
 end
 
 failed_scenarios = finished_scenarios.reject(&:success?)
@@ -232,7 +233,7 @@ else
   puts "\nFailed scenarios:\n\n"
 
   failed_scenarios.each do |scenario|
-    puts "\e[#{31}m#{'[FAILED]'}\e[0m #{scenario.name}"
+    puts "\e[31m[FAILED]\e[0m #{scenario.name}"
   end
 
   puts

data/bin/verify_topics_naming

@@ -8,18 +8,18 @@
 #
 # It also ensures we have one convention that we can follow.
 
-require 'bundler'
+require "bundler"
 Bundler.setup(:default, :test, :integrations)
-require 'rdkafka'
+require "rdkafka"
 
-admin = Rdkafka::Config.new('bootstrap.servers': 'localhost:9092').admin
+admin = Rdkafka::Config.new("bootstrap.servers": "localhost:9092").admin
 
 invalid = admin
-          .metadata
-          .topics
-          .map { |topic| topic[:topic_name] }
-          .select { |topic| !topic.start_with?('it-') }
-          .select { |topic| topic.length <= 6 }
+  .metadata
+  .topics
+  .map { |topic| topic[:topic_name] }
+  .select { |topic| !topic.start_with?("it-") }
+  .select { |topic| topic.length <= 6 }
 
 admin.close
 

data/config/locales/errors.yml

@@ -26,6 +26,19 @@ en:
       non_reloadable_errors_format: must be an array of symbols
       oauth.token_provider_listener_format: 'must be false or respond to #on_oauthbearer_token_refresh'
       idle_disconnect_timeout_format: 'must be an integer that is equal to 0 or bigger than 30 000 (30 seconds)'
+      statistics_decorator_format: 'must respond to #call'
+      polling.mode_format: must be :thread or :fd
+      polling.poller_format: must be nil or a WaterDrop::Polling::Poller instance
+      polling.poller_poller_only_with_fd_mode: can only be set when polling.mode is :fd
+      polling.fd.max_time_format: must be an integer that is equal or bigger than 1
+      polling.fd.periodic_poll_interval_format: must be an integer that is equal or bigger than 1
+
+    poller:
+      thread_priority_format: must be an integer between -3 and 3
+      poll_timeout_format: must be an integer that is equal or bigger than 1
+      backoff_min_format: must be an integer that is equal or bigger than 1
+      backoff_max_format: must be an integer that is equal or bigger than 1
+      backoff_max_must_be_gte_backoff_min: backoff_max must be greater than or equal to backoff_min
 
     variant:
       missing: must be present

data/docker-compose.oauth.yml

@@ -0,0 +1,56 @@
+services:
+  keycloak:
+    image: quay.io/keycloak/keycloak:26.5
+    container_name: keycloak
+    command: start-dev --import-realm
+    ports:
+      - "8080:8080"
+    environment:
+      KEYCLOAK_ADMIN: admin
+      KEYCLOAK_ADMIN_PASSWORD: admin
+    volumes:
+      - ./test/integrations/oauth_token_refresh/keycloak-realm.json:/opt/keycloak/data/import/realm.json
+    healthcheck:
+      test: ["CMD-SHELL", "exec 3<>/dev/tcp/localhost/8080 && echo -e 'GET /realms/kafka HTTP/1.1\\r\\nHost: localhost\\r\\n\\r\\n' >&3 && timeout 1 cat <&3 | grep -q '200 OK'"]
+      interval: 10s
+      timeout: 10s
+      retries: 30
+      start_period: 90s
+
+  kafka-oauth:
+    image: confluentinc/cp-kafka:8.2.0
+    container_name: kafka-oauth
+    depends_on:
+      keycloak:
+        condition: service_healthy
+    ports:
+      - "9094:9094"
+    environment:
+      CLUSTER_ID: kafka-oauth-cluster-1
+      KAFKA_BROKER_ID: 1
+      KAFKA_PROCESS_ROLES: broker,controller
+      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka-oauth:9093
+      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,SASL_PLAINTEXT://:9094
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-oauth:9092,SASL_PLAINTEXT://127.0.0.1:9094
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,SASL_PLAINTEXT:SASL_PLAINTEXT
+      KAFKA_SASL_ENABLED_MECHANISMS: OAUTHBEARER
+      KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAINTEXT
+      # Per-listener callback handler
+      KAFKA_LISTENER_NAME_SASL__PLAINTEXT_OAUTHBEARER_SASL_SERVER_CALLBACK_HANDLER_CLASS: org.apache.kafka.common.security.oauthbearer.OAuthBearerValidatorCallbackHandler
+      KAFKA_LISTENER_NAME_SASL__PLAINTEXT_OAUTHBEARER_SASL_JAAS_CONFIG: |
+        org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule required;
+      # Global OAUTHBEARER settings for token validation
+      KAFKA_SASL_OAUTHBEARER_JWKS_ENDPOINT_URL: http://keycloak:8080/realms/kafka/protocol/openid-connect/certs
+      KAFKA_SASL_OAUTHBEARER_EXPECTED_AUDIENCE: kafka
+      # Note: Not setting expected issuer - Keycloak uses request URL which varies
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      # Required by Confluent image when SASL is enabled
+      # Also allow HTTP URLs for JWKS endpoint (Kafka 3.0+ blocks non-HTTPS by default)
+      KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf -Dorg.apache.kafka.sasl.oauthbearer.allowed.urls=http://keycloak:8080/realms/kafka/protocol/openid-connect/certs"
+    volumes:
+      - ./test/integrations/oauth_token_refresh/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf:ro

data/docker-compose.yml

@@ -1,7 +1,7 @@
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:8.0.0
+    image: confluentinc/cp-kafka:8.2.0
 
     ports:
       - 9092:9092

data/lib/waterdrop/clients/dummy.rb

@@ -57,6 +57,15 @@ module WaterDrop
         Handle.new(topic.to_s, partition, @counters["#{topic}#{partition}"] += 1)
       end
 
+      # Returns 0 as dummy client doesn't queue any real messages
+      #
+      # @return [Integer] always 0
+      def queue_size
+        0
+      end
+
+      alias_method :queue_length, :queue_size
+
       # @param _args [Object] anything really, this dummy is suppose to support anything
       def respond_to_missing?(*_args)
         true

data/lib/waterdrop/clients/rdkafka.rb

@@ -14,12 +14,23 @@ module WaterDrop
           kafka_config = producer.config.kafka.to_h
           monitor = producer.config.monitor
 
-          client = ::Rdkafka::Config.new(kafka_config).producer(native_kafka_auto_start: false)
+          # When FD polling is enabled, we disable the native librdkafka polling thread
+          # and use our own Ruby-based poller instead
+          producer_options = { native_kafka_auto_start: false }
+
+          producer_options[:run_polling_thread] = false if producer.fd_polling?
+
+          client = ::Rdkafka::Config.new(kafka_config).producer(**producer_options)
 
           # Register statistics runner for this particular type of callbacks
           ::Karafka::Core::Instrumentation.statistics_callbacks.add(
             producer.id,
-            Instrumentation::Callbacks::Statistics.new(producer.id, client.name, monitor)
+            Instrumentation::Callbacks::Statistics.new(
+              producer.id,
+              client.name,
+              monitor,
+              producer.config.statistics_decorator
+            )
           )
 
           # Register error tracking callback
@@ -53,8 +64,13 @@ module WaterDrop
 
           client.start
 
+          # Register with poller if FD polling is enabled
+          # Uses the producer's configured poller (custom or global singleton)
+          # This must happen after client.start to ensure the client is ready
+          producer.poller.register(producer, client) if producer.fd_polling?
+
           # Switch to the transactional mode if user provided the transactional id
-          client.init_transactions if kafka_config.key?(:'transactional.id')
+          client.init_transactions if kafka_config.key?(:"transactional.id")
 
           client
         end

data/lib/waterdrop/config.rb

@@ -9,18 +9,18 @@ module WaterDrop
 
     # Defaults for kafka settings, that will be overwritten only if not present already
     KAFKA_DEFAULTS = {
-      'client.id': 'waterdrop',
+      "client.id": "waterdrop",
       # emit librdkafka statistics every five seconds. This is used in instrumentation.
       # When disabled, part of metrics will not be published and available.
-      'statistics.interval.ms': 5_000,
+      "statistics.interval.ms": 5_000,
       # We set it to a value that is lower than `max_wait_timeout` to have a final verdict upon
       # sync delivery
-      'message.timeout.ms': 50_000,
+      "message.timeout.ms": 150_000,
       # Must be more or equal to `message.timeout.ms` defaults
-      'transaction.timeout.ms': 55_000,
+      "transaction.timeout.ms": 165_000,
       # Lowers latency. Default in newer librdkafka but we want to make sure it is shipped to
       # users despite what librdkafka they run on
-      'socket.nagle.disable': true
+      "socket.nagle.disable": true
     }.freeze
 
     private_constant :KAFKA_DEFAULTS
@@ -53,7 +53,7 @@ module WaterDrop
     setting :max_payload_size, default: 1_000_012
     # option [Integer] Wait that long for the delivery report or raise an error if this takes
     #   longer than the timeout ms.
-    setting :max_wait_timeout, default: 60_000
+    setting :max_wait_timeout, default: 180_000
     # option [Boolean] should we upon detecting full librdkafka queue backoff and retry or should
     #   we raise an exception.
     #   When this is set to `true`, upon full queue, we won't raise an error. There will be error
@@ -116,6 +116,23 @@ module WaterDrop
     #   prevent overly aggressive disconnections.
     setting :idle_disconnect_timeout, default: 0
 
+    # option [Karafka::Core::Monitoring::StatisticsDecorator] decorator instance used to compute
+    #   deltas (_d) and freeze durations (_fd) on raw librdkafka statistics. The default is
+    #   pre-configured with `only_keys` covering keys used by the built-in Datadog metrics
+    #   listener and `excluded_keys` skipping subtrees not needed by producers (topics, broker
+    #   window stats). Users who need additional decorated keys or full decoration can provide
+    #   a custom decorator instance.
+    setting(
+      :statistics_decorator,
+      default: false,
+      constructor: lambda { |decorator|
+        decorator || ::Karafka::Core::Monitoring::StatisticsDecorator.new(
+          only_keys: %w[tx txretries txerrs rxerrs],
+          excluded_keys: %w[int_latency outbuf_latency rtt throttle req toppars topics]
+        )
+      }
+    )
+
     # option [Boolean] should we send messages. Setting this to false can be really useful when
     #   testing and or developing because when set to false, won't actually ping Kafka but will
     #   run all the validations, etc
@@ -140,6 +157,33 @@ module WaterDrop
       setting :token_provider_listener, default: false
     end
 
+    # Namespace for polling configuration
+    setting :polling do
+      # option [Symbol] Polling mode for handling producer callbacks
+      # :fd - uses a single global Ruby thread with IO.select-based multiplexing (default)
+      # :thread - uses librdkafka's native background polling threads
+      setting :mode, default: :fd
+
+      # option [WaterDrop::Polling::Poller, nil] Custom poller instance for isolation
+      # When nil (default), uses the global singleton poller shared by all FD-mode producers
+      # When set to a Poller instance, uses that dedicated poller for this producer only
+      # This allows isolating producers from each other's callback processing
+      # Only valid when mode is :fd
+      setting :poller, default: nil
+
+      # Namespace for FD-based polling configuration (only used when mode is :fd)
+      setting :fd do
+        # option [Integer] Max milliseconds to poll a single producer before rotating to the next
+        # This prevents any single producer from monopolizing the polling thread
+        setting :max_time, default: 100
+
+        # option [Integer] Interval in milliseconds for periodic polling of idle producers
+        # When one producer is busy, other producers still need to be polled periodically
+        # to ensure OAuth token refresh and statistics callbacks fire on time
+        setting :periodic_poll_interval, default: 1_000
+      end
+    end
+
     # Configuration method
     # @yield Runs a block of code providing a config singleton instance to it
     # @yieldparam [WaterDrop::Config] WaterDrop config instance