waterdrop 2.8.11 → 2.8.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3dc4cc4d1928a447c4e6c3ab9d268a87c0d3a462b1b9d62685fe51168842c0e1
-  data.tar.gz: b82300c02b3d774aba5a2a354f7b9d0d11f2d08738ccf66348185f1a4bbd1605
+  metadata.gz: a0890fcd73147d293340b55cca58c975d81726208c609be1f338ce56d87b4d18
+  data.tar.gz: 7a2743c068af9936186fa852d6e663c85c172aa93de76b16aaf09e2c1de24a25
 SHA512:
-  metadata.gz: a2f0d2befaa70b25bf756fdee15757328cfa0d2a642243100ae89421c0b0d65ce7b3b3d94d630f90cf040b7bb3ce2218aaee8200edfdfc96164038fee6d79e9b
-  data.tar.gz: 2b5a2fae3f060435c697024aab1c064342d9919b03304b8c29a1ee25c0d85533a0b43af5184c4dd8ceb5d49377178536ca6b0306af473ed4876dc98f712bf7c1
+  metadata.gz: 66136311cb010a1648a0a5440cd930fcb96bb75fe30575d5c14effb8550a7cea4295dda98828790ac567ec9f3346dde7901c11228f6ca251fdf454a5354fad9d
+  data.tar.gz: a701a63fe3a171084b63ac46ca9f66d75572b7efbfed0b794884df0fdb2b272404a6594ba239f16956cdc9c5fbecb31dbb5574fc0e5c1f12225ce61514c776a3

data/CHANGELOG.md

@@ -1,5 +1,21 @@
 # WaterDrop changelog
 
+## 2.8.12 (2025-10-10)
+- [Enhancement] Introduce `reload_on_idempotent_fatal_error` to automatically reload librdkafka producer after fatal errors on idempotent (non-transactional) producers.
+- [Enhancement] Add configurable backoff and retry limits for fatal error recovery to prevent infinite reload loops:
+  - `wait_backoff_on_idempotent_fatal_error` (default: 5000ms) - backoff before retrying after idempotent fatal error reload
+  - `max_attempts_on_idempotent_fatal_error` (default: 5) - max reload attempts for idempotent fatal errors
+  - `wait_backoff_on_transaction_fatal_error` (default: 1000ms) - backoff after transactional fatal error reload
+  - `max_attempts_on_transaction_fatal_error` (default: 10) - max reload attempts for transactional fatal errors
+- [Enhancement] Ensure `error.occurred` is instrumented before idempotent fatal error reload for visibility.
+- [Enhancement] Automatically reset fatal error reload attempts counter on successful produce/transaction to allow recovery.
+- [Refactor] Extract idempotence-related logic into separate `WaterDrop::Producer::Idempotence` module.
+- [Refactor] Initialize `@idempotent` and `@transactional` instance variables in Producer#initialize for consistent Ruby object shapes optimization.
+- [Refactor] Add `idempotent_reloadable?` and `idempotent_retryable?` methods to encapsulate idempotent fatal error reload checks.
+- [Refactor] Add `transactional_retryable?` method to encapsulate transactional fatal error reload retry checks.
+- [Fix] Waterdrop `config.kafka` errors on frozen hash.
+- [Fix] `Producer#transactional?` method now correctly computes transactional status when `@transactional` is initialized to nil.
+
 ## 2.8.11 (2025-09-27)
 - [Enhancement] Provide fast-track for middleware-less flows (20% faster) for single message, 5000x faster for batches.
 - [Enhancement] Optimize middlewares application by around 20%.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.8.11)
+    waterdrop (2.8.12)
       karafka-core (>= 2.4.9, < 3.0.0)
       karafka-rdkafka (>= 0.20.0)
       zeitwerk (~> 2.3)

data/config/locales/errors.yml

@@ -17,7 +17,12 @@ en:
       wait_timeout_on_queue_full_format: must be a numeric that is equal or bigger to 0
       wait_backoff_on_transaction_command_format: must be a numeric that is equal or bigger to 0
       max_attempts_on_transaction_command_format: must be an integer that is equal or bigger than 1
+      reload_on_idempotent_fatal_error_format: must be boolean
+      wait_backoff_on_idempotent_fatal_error_format: must be a numeric that is equal or bigger to 0
+      max_attempts_on_idempotent_fatal_error_format: must be an integer that is equal or bigger than 1
       reload_on_transaction_fatal_error_format: must be boolean
+      wait_backoff_on_transaction_fatal_error_format: must be a numeric that is equal or bigger to 0
+      max_attempts_on_transaction_fatal_error_format: must be an integer that is equal or bigger than 1
       oauth.token_provider_listener_format: 'must be false or respond to #on_oauthbearer_token_refresh'
       idle_disconnect_timeout_format: 'must be an integer that is equal to 0 or bigger than 30 000 (30 seconds)'
 

data/lib/waterdrop/config.rb

@@ -61,7 +61,7 @@ module WaterDrop
     #   recoverable, in a high number it still may mean issues.
     #   Waiting is one of the recommended strategies.
     setting :wait_on_queue_full, default: true
-    # option [Integer] how long (in seconds) should we backoff before a retry when queue is full
+    # option [Integer] how long (in ms) should we backoff before a retry when queue is full
     #   The retry will happen with the same message and backoff should give us some time to
     #   dispatch previously buffered messages.
     setting :wait_backoff_on_queue_full, default: 100
@@ -79,6 +79,22 @@ module WaterDrop
     # to keep going or should we stop. Since we will open a new instance and the failed transaction
     # anyhow rolls back, we should be able to safely reload.
     setting :reload_on_transaction_fatal_error, default: true
+    # option [Boolean] When a fatal error occurs on idempotent producer, should we automatically
+    #   close and recreate the underlying producer to recover and continue sending messages. This
+    #   allows automatic recovery from fatal librdkafka errors in idempotent producers.
+    setting :reload_on_idempotent_fatal_error, default: false
+    # option [Numeric] How long to wait (in ms) before retrying after reloading on idempotent
+    #   fatal error. This backoff prevents rapid reload loops from overloading the system.
+    setting :wait_backoff_on_idempotent_fatal_error, default: 5_000
+    # option [Integer] How many times to attempt reloading on idempotent fatal error before giving
+    #   up. This prevents infinite reload loops if the producer never recovers.
+    setting :max_attempts_on_idempotent_fatal_error, default: 5
+    # option [Numeric] How long to wait (in ms) before continuing after reloading on transactional
+    #   fatal error. This backoff prevents rapid reload loops from overloading the system.
+    setting :wait_backoff_on_transaction_fatal_error, default: 1_000
+    # option [Integer] How many times to attempt reloading on transactional fatal error before
+    #   giving up. This prevents infinite reload loops if the producer never recovers.
+    setting :max_attempts_on_transaction_fatal_error, default: 10
     # option [Integer] Idle disconnect timeout in milliseconds. When set to 0, idle disconnection
     #   is disabled. When set to a positive value, WaterDrop will automatically disconnect
     #   producers that haven't sent any messages for the specified time period. This helps preserve
@@ -134,6 +150,10 @@ module WaterDrop
     # to overwrite the whole hash if they want to
     # @param config [Karafka::Core::Configurable::Node] config of this producer
     def merge_kafka_defaults!(config)
+      # Duplicate the kafka config so we can mutate it internally if needed without polluting the
+      # user provided object
+      config.kafka = config.kafka.dup
+
       KAFKA_DEFAULTS.each do |key, value|
         next if config.kafka.key?(key)
 

data/lib/waterdrop/connection_pool.rb

@@ -84,7 +84,6 @@ module WaterDrop
 
       # Executes a block with a producer from the global pool
       #
-      # @param block [Proc] Block to execute with a producer
       # @yield [producer] Producer from the global pool
       # @return [Object] Result of the block
       # @raise [RuntimeError] If no global pool is configured
@@ -93,10 +92,10 @@ module WaterDrop
       #   WaterDrop::ConnectionPool.with do |producer|
       #     producer.produce_sync(topic: 'events', payload: 'data')
       #   end
-      def with(&block)
+      def with(...)
         raise 'No global connection pool configured. Call setup first.' unless @default_pool
 
-        @default_pool.with(&block)
+        @default_pool.with(...)
       end
 
       # Get statistics about the global pool
@@ -154,7 +153,6 @@ module WaterDrop
       # Execute a transaction with a producer from the global connection pool
       # Only available when connection pool is configured
       #
-      # @param block [Proc] Block to execute within a transaction
       # @yield [producer] Producer from the global pool with an active transaction
       # @return [Object] Result of the block
       # @raise [RuntimeError] If no global pool is configured
@@ -164,10 +162,10 @@ module WaterDrop
       #     producer.produce(topic: 'events', payload: 'data1')
       #     producer.produce(topic: 'events', payload: 'data2')
       #   end
-      def transaction(&block)
+      def transaction(...)
         raise 'No global connection pool configured. Call setup first.' unless @default_pool
 
-        @default_pool.transaction(&block)
+        @default_pool.transaction(...)
       end
 
       private
@@ -298,7 +296,6 @@ module WaterDrop
     # Execute a block with a producer from the global connection pool
     # Only available when connection pool is configured
     #
-    # @param block [Proc] Block to execute with a producer
     # @yield [producer] Producer from the global pool
     # @return [Object] Result of the block
     #
@@ -306,14 +303,13 @@ module WaterDrop
     #   WaterDrop.with do |producer|
     #     producer.produce_sync(topic: 'events', payload: 'data')
     #   end
-    def with(&block)
-      ConnectionPool.with(&block)
+    def with(...)
+      ConnectionPool.with(...)
     end
 
     # Execute a transaction with a producer from the global connection pool
     # Only available when connection pool is configured
     #
-    # @param block [Proc] Block to execute within a transaction
     # @yield [producer] Producer from the global pool with an active transaction
     # @return [Object] Result of the block
     #
@@ -322,8 +318,8 @@ module WaterDrop
     #     producer.produce(topic: 'events', payload: 'data1')
     #     producer.produce(topic: 'events', payload: 'data2')
     #   end
-    def transaction(&block)
-      ConnectionPool.transaction(&block)
+    def transaction(...)
+      ConnectionPool.transaction(...)
     end
 
     # Access the global connection pool

data/lib/waterdrop/contracts/config.rb

@@ -5,10 +5,8 @@ module WaterDrop
     # Contract with validation rules for WaterDrop configuration details
     class Config < ::Karafka::Core::Contractable::Contract
       configure do |config|
-        config.error_messages = YAML.safe_load(
-          File.read(
-            File.join(WaterDrop.gem_root, 'config', 'locales', 'errors.yml')
-          )
+        config.error_messages = YAML.safe_load_file(
+          File.join(WaterDrop.gem_root, 'config', 'locales', 'errors.yml')
         ).fetch('en').fetch('validations').fetch('config')
       end
 
@@ -27,6 +25,11 @@ module WaterDrop
       required(:wait_backoff_on_transaction_command) { |val| val.is_a?(Numeric) && val >= 0 }
       required(:max_attempts_on_transaction_command) { |val| val.is_a?(Integer) && val >= 1 }
       required(:reload_on_transaction_fatal_error) { |val| [true, false].include?(val) }
+      required(:reload_on_idempotent_fatal_error) { |val| [true, false].include?(val) }
+      required(:wait_backoff_on_idempotent_fatal_error) { |val| val.is_a?(Numeric) && val >= 0 }
+      required(:max_attempts_on_idempotent_fatal_error) { |val| val.is_a?(Integer) && val >= 1 }
+      required(:wait_backoff_on_transaction_fatal_error) { |val| val.is_a?(Numeric) && val >= 0 }
+      required(:max_attempts_on_transaction_fatal_error) { |val| val.is_a?(Integer) && val >= 1 }
       required(:idle_disconnect_timeout) do |val|
         val.is_a?(Integer) && (val.zero? || val >= 30_000)
       end
@@ -42,13 +45,11 @@ module WaterDrop
       virtual do |config, errors|
         next true unless errors.empty?
 
-        errors = []
-
-        config
-          .fetch(:kafka)
-          .keys
-          .reject { |key| key.is_a?(Symbol) }
-          .each { |key| errors << [[:kafka, key], :kafka_key_must_be_a_symbol] }
+        errors = config
+                 .fetch(:kafka)
+                 .keys
+                 .reject { |key| key.is_a?(Symbol) }
+                 .map { |key| [[:kafka, key], :kafka_key_must_be_a_symbol] }
 
         errors
       end

data/lib/waterdrop/contracts/message.rb

@@ -6,10 +6,8 @@ module WaterDrop
     # we provide to producer are valid and usable
     class Message < ::Karafka::Core::Contractable::Contract
       configure do |config|
-        config.error_messages = YAML.safe_load(
-          File.read(
-            File.join(WaterDrop.gem_root, 'config', 'locales', 'errors.yml')
-          )
+        config.error_messages = YAML.safe_load_file(
+          File.join(WaterDrop.gem_root, 'config', 'locales', 'errors.yml')
         ).fetch('en').fetch('validations').fetch('message')
       end
 
@@ -49,7 +47,7 @@ module WaterDrop
 
           # Headers can be either strings or arrays of strings
           next if value.is_a?(String)
-          next if value.is_a?(Array) && value.all? { |value| value.is_a?(String) }
+          next if value.is_a?(Array) && value.all?(String)
 
           errors << [%i[headers], :invalid_value_type]
         end

data/lib/waterdrop/contracts/transactional_offset.rb

@@ -6,10 +6,8 @@ module WaterDrop
     # and match our expectations
     class TransactionalOffset < ::Karafka::Core::Contractable::Contract
       configure do |config|
-        config.error_messages = YAML.safe_load(
-          File.read(
-            File.join(WaterDrop.gem_root, 'config', 'locales', 'errors.yml')
-          )
+        config.error_messages = YAML.safe_load_file(
+          File.join(WaterDrop.gem_root, 'config', 'locales', 'errors.yml')
         ).fetch('en').fetch('validations').fetch('transactional_offset')
       end
 

data/lib/waterdrop/contracts/variant.rb

@@ -27,10 +27,8 @@ module WaterDrop
       private_constant :TOPIC_CONFIG_KEYS, :BOOLEANS
 
       configure do |config|
-        config.error_messages = YAML.safe_load(
-          File.read(
-            File.join(WaterDrop.gem_root, 'config', 'locales', 'errors.yml')
-          )
+        config.error_messages = YAML.safe_load_file(
+          File.join(WaterDrop.gem_root, 'config', 'locales', 'errors.yml')
         ).fetch('en').fetch('validations').fetch('variant')
       end
 
@@ -41,13 +39,11 @@ module WaterDrop
       virtual do |config, errors|
         next true unless errors.empty?
 
-        errors = []
-
-        config
-          .fetch(:topic_config)
-          .keys
-          .reject { |key| key.is_a?(Symbol) }
-          .each { |key| errors << [[:kafka, key], :kafka_key_must_be_a_symbol] }
+        errors = config
+                 .fetch(:topic_config)
+                 .keys
+                 .reject { |key| key.is_a?(Symbol) }
+                 .map { |key| [[:kafka, key], :kafka_key_must_be_a_symbol] }
 
         errors
       end
@@ -56,13 +52,11 @@ module WaterDrop
       virtual do |config, errors|
         next true unless errors.empty?
 
-        errors = []
-
-        config
-          .fetch(:topic_config)
-          .keys
-          .reject { |key| TOPIC_CONFIG_KEYS.include?(key) }
-          .each { |key| errors << [[:kafka, key], :kafka_key_not_per_topic] }
+        errors = config
+                 .fetch(:topic_config)
+                 .keys
+                 .reject { |key| TOPIC_CONFIG_KEYS.include?(key) }
+                 .map { |key| [[:kafka, key], :kafka_key_not_per_topic] }
 
         errors
       end
@@ -75,13 +69,11 @@ module WaterDrop
         # Relevant only for the transactional producer
         next true unless config.fetch(:transactional)
 
-        errors = []
-
-        config
-          .fetch(:topic_config)
-          .keys
-          .select { |key| key.to_s.include?('acks') }
-          .each { |key| errors << [[:kafka, key], :kafka_key_acks_not_changeable] }
+        errors = config
+                 .fetch(:topic_config)
+                 .keys
+                 .select { |key| key.to_s.include?('acks') }
+                 .map { |key| [[:kafka, key], :kafka_key_acks_not_changeable] }
 
         errors
       end
@@ -92,13 +84,11 @@ module WaterDrop
         # Relevant only for the transactional producer
         next true unless config.fetch(:idempotent)
 
-        errors = []
-
-        config
-          .fetch(:topic_config)
-          .keys
-          .select { |key| key.to_s.include?('acks') }
-          .each { |key| errors << [[:kafka, key], :kafka_key_acks_not_changeable] }
+        errors = config
+                 .fetch(:topic_config)
+                 .keys
+                 .select { |key| key.to_s.include?('acks') }
+                 .map { |key| [[:kafka, key], :kafka_key_acks_not_changeable] }
 
         errors
       end

data/lib/waterdrop/instrumentation/class_monitor.rb

@@ -11,7 +11,7 @@ module WaterDrop
         notifications_bus = WaterDrop::Instrumentation::ClassNotifications.new,
         namespace = nil
       )
-        super(notifications_bus, namespace)
+        super
       end
     end
   end

data/lib/waterdrop/instrumentation/monitor.rb

@@ -13,7 +13,7 @@ module WaterDrop
         notifications_bus = WaterDrop::Instrumentation::Notifications.new,
         namespace = nil
       )
-        super(notifications_bus, namespace)
+        super
       end
     end
   end

data/lib/waterdrop/instrumentation/vendors/datadog/metrics_listener.rb

@@ -58,10 +58,9 @@ module WaterDrop
             setup(&block) if block
           end
 
-          # @param block [Proc] configuration block
           # @note We define this alias to be consistent with `WaterDrop#setup`
-          def setup(&block)
-            configure(&block)
+          def setup(...)
+            configure(...)
           end
 
           # Hooks up to WaterDrop instrumentation for emitted statistics
@@ -92,6 +91,16 @@ module WaterDrop
             produced_sync
             produced_async
           ].each do |event_scope|
+            # @example
+            #   def on_message_produced_sync(event)
+            #     report_message(event[:message][:topic], :produced_sync)
+            #   end
+            #
+            #   def on_messages_produced_sync(event)
+            #     event[:messages].each do |message|
+            #       report_message(message[:topic], :produced_sync)
+            #     end
+            #   end
             class_eval <<~METHODS, __FILE__, __LINE__ + 1
               # @param event [Karafka::Core::Monitoring::Event]
               def on_message_#{event_scope}(event)
@@ -112,6 +121,14 @@ module WaterDrop
             message_buffered
             messages_buffered
           ].each do |event_scope|
+            # @example
+            #   def on_message_buffered(event)
+            #     histogram(
+            #       'buffer.size',
+            #       event[:buffer].size,
+            #       tags: default_tags
+            #     )
+            #   end
             class_eval <<~METHODS, __FILE__, __LINE__ + 1
               # @param event [Karafka::Core::Monitoring::Event]
               def on_#{event_scope}(event)
@@ -130,6 +147,12 @@ module WaterDrop
             flushed_sync
             flushed_async
           ].each do |event_scope|
+            # @example
+            #   def on_buffer_flushed_sync(event)
+            #     event[:messages].each do |message|
+            #       report_message(message[:topic], :flushed_sync)
+            #     end
+            #   end
             class_eval <<~METHODS, __FILE__, __LINE__ + 1
               # @param event [Karafka::Core::Monitoring::Event]
               def on_buffer_#{event_scope}(event)
@@ -149,6 +172,13 @@ module WaterDrop
             increment
             decrement
           ].each do |metric_type|
+            # @example
+            #   def count(key, *args)
+            #     client.count(
+            #       namespaced_metric(key),
+            #       *args
+            #     )
+            #   end
             class_eval <<~METHODS, __FILE__, __LINE__ + 1
               def #{metric_type}(key, *args)
                 client.#{metric_type}(

data/lib/waterdrop/producer/idempotence.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  class Producer
+    # Component for idempotent producer operations and error recovery
+    module Idempotence
+      # @return [Boolean] true if current producer is idempotent
+      def idempotent?
+        # Every transactional producer is idempotent by default always
+        return true if transactional?
+        return @idempotent unless @idempotent.nil?
+
+        @idempotent = config.kafka.to_h.fetch(:'enable.idempotence', false)
+      end
+
+      # Checks if the given error should trigger an idempotent producer reload
+      #
+      # @param error [Rdkafka::RdkafkaError] the error to check
+      # @return [Boolean] true if the error should trigger a reload
+      #
+      # @note Returns true only if all of the following conditions are met:
+      #   - Error is fatal
+      #   - Producer is idempotent
+      #   - Producer is not transactional
+      #   - reload_on_idempotent_fatal_error config is enabled
+      #   - Error is not in the NON_RELOADABLE_FATAL_ERRORS list
+      def idempotent_reloadable?(error)
+        return false unless error.fatal?
+        return false unless idempotent?
+        return false if transactional?
+        return false unless config.reload_on_idempotent_fatal_error
+        return false if NON_RELOADABLE_FATAL_ERRORS.include?(error.code)
+
+        true
+      end
+
+      # Checks if we can still retry reloading after an idempotent fatal error
+      #
+      # @return [Boolean] true if we haven't exceeded the max reload attempts yet
+      def idempotent_retryable?
+        @idempotent_fatal_error_attempts < config.max_attempts_on_idempotent_fatal_error
+      end
+
+      private
+
+      # Reloads the underlying client instance when a fatal error occurs on an idempotent producer
+      #
+      # This method handles fatal errors that can occur in idempotent (non-transactional) producers
+      # When a fatal error is detected, it will flush pending messages, purge the queue, close the
+      # old client, and create a new client instance to continue operations.
+      #
+      # @param attempt [Integer] the current reload attempt number
+      #
+      # @note This is only called for idempotent, non-transactional producers when
+      #   `reload_on_idempotent_fatal_error` is enabled
+      # @note After reload, the producer will automatically retry the failed operation
+      def idempotent_reload_client_on_fatal_error(attempt)
+        @operating_mutex.synchronize do
+          @monitor.instrument(
+            'producer.reloaded',
+            producer_id: id,
+            attempt: attempt
+          ) do
+            @client.flush(current_variant.max_wait_timeout)
+            purge
+            @client.close
+            @client = Builder.new.call(self, @config)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/waterdrop/producer/status.rb

@@ -38,6 +38,14 @@ module WaterDrop
       end
 
       LIFECYCLE.each do |state|
+        # @example
+        #   def initial?
+        #     @current == :initial
+        #   end
+        #
+        #   def initial!
+        #     @current = :initial
+        #   end
         module_eval <<-RUBY, __FILE__, __LINE__ + 1
           # @return [Boolean] true if current status is as we want, otherwise false
           def #{state}?

data/lib/waterdrop/producer/transactions.rb

@@ -4,16 +4,10 @@ module WaterDrop
   class Producer
     # Transactions related producer functionalities
     module Transactions
-      # We should never reload producer if it was fenced, otherwise we could end up with some sort
-      # of weird race-conditions
-      NON_RELOADABLE_ERRORS = %i[
-        fenced
-      ].freeze
-
       # Contract to validate that input for transactional offset storage is correct
       CONTRACT = Contracts::TransactionalOffset.new
 
-      private_constant :CONTRACT, :NON_RELOADABLE_ERRORS
+      private_constant :CONTRACT
 
       # Creates a transaction.
       #
@@ -99,6 +93,9 @@ module WaterDrop
               transactional_instrument(:committed) { client.commit_transaction }
             end
 
+            # Reset attempts counter on successful transaction commit
+            @transaction_fatal_error_attempts = 0
+
             result
           # We need to handle any interrupt including critical in order not to have the transaction
           # running. This will also handle things like `IRB::Abort`
@@ -135,11 +132,18 @@ module WaterDrop
 
       # @return [Boolean] Is this producer a transactional one
       def transactional?
-        return @transactional if instance_variable_defined?(:'@transactional')
+        return @transactional unless @transactional.nil?
 
         @transactional = config.kafka.to_h.key?(:'transactional.id')
       end
 
+      # Checks if we can still retry reloading after a transactional fatal error
+      #
+      # @return [Boolean] true if we haven't exceeded the max reload attempts yet
+      def transactional_retryable?
+        @transaction_fatal_error_attempts < config.max_attempts_on_transaction_fatal_error
+      end
+
       # Marks given message as consumed inside of a transaction.
       #
       # @param consumer [#consumer_group_metadata_pointer] any consumer from which we can obtain
@@ -188,18 +192,16 @@ module WaterDrop
       # This allows us to simplify the async and sync batch dispatchers because we can ensure that
       # their internal dispatches will be wrapped only with a single transaction and not
       # a transaction per message
-      # @param block [Proc] code we want to run
-      def with_transaction_if_transactional(&block)
-        transactional? ? transaction(&block) : yield
+      def with_transaction_if_transactional(...)
+        transactional? ? transaction(...) : yield
       end
 
       # Instruments the transactional operation with producer id
       #
       # @param key [Symbol] transaction operation key
       # @param details [Hash] additional instrumentation details
-      # @param block [Proc] block to run inside the instrumentation or nothing if not given
-      def transactional_instrument(key, details = EMPTY_HASH, &block)
-        @monitor.instrument("transaction.#{key}", details.merge(producer_id: id), &block)
+      def transactional_instrument(key, details = EMPTY_HASH, ...)
+        @monitor.instrument("transaction.#{key}", details.merge(producer_id: id), ...)
       end
 
       # Error handling for transactional operations is a bit special. There are three types of
@@ -279,12 +281,19 @@ module WaterDrop
 
         return unless rd_error.is_a?(Rdkafka::RdkafkaError)
         return unless config.reload_on_transaction_fatal_error
-        return if NON_RELOADABLE_ERRORS.include?(rd_error.code)
+        return if NON_RELOADABLE_FATAL_ERRORS.include?(rd_error.code)
+
+        # Check if we've exceeded max reload attempts
+        return unless transactional_retryable?
+
+        # Increment attempts before reload
+        @transaction_fatal_error_attempts += 1
 
         @operating_mutex.synchronize do
           @monitor.instrument(
             'producer.reloaded',
-            producer_id: id
+            producer_id: id,
+            attempt: @transaction_fatal_error_attempts
           ) do
             @client.flush(current_variant.max_wait_timeout)
             purge
@@ -292,6 +301,9 @@ module WaterDrop
             @client = Builder.new.call(self, @config)
           end
         end
+
+        # Wait before continuing to avoid rapid reload loops
+        sleep(config.wait_backoff_on_transaction_fatal_error / 1_000.0)
       end
     end
   end

data/lib/waterdrop/producer/variant.rb

@@ -74,6 +74,15 @@ module WaterDrop
         Transactions
       ].each do |scope|
         scope.instance_methods(false).each do |method_name|
+          # @example
+          #   def produce_async(*args, &block)
+          #     ref = Fiber.current.waterdrop_clients ||= {}
+          #     ref[@producer.id] = self
+          #
+          #     @producer.produce_async(*args, &block)
+          #   ensure
+          #     ref[@producer.id] = nil
+          #   end
           class_eval <<-RUBY, __FILE__, __LINE__ + 1
             def #{method_name}(*args, &block)
               ref = Fiber.current.waterdrop_clients ||= {}

data/lib/waterdrop/producer.rb

@@ -8,6 +8,7 @@ module WaterDrop
     include Async
     include Buffer
     include Transactions
+    include Idempotence
     include ClassMonitor
     include ::Karafka::Core::Helpers::Time
     include ::Karafka::Core::Taggable
@@ -21,13 +22,21 @@ module WaterDrop
       Rdkafka::Producer::DeliveryHandle::WaitTimeoutError
     ].freeze
 
+    # We should never reload producer on certain fatal errors as they may indicate state that
+    # cannot be recovered by simply recreating the client
+    NON_RELOADABLE_FATAL_ERRORS = %i[
+      fenced
+    ].freeze
+
     # Empty hash to save on memory allocations
     EMPTY_HASH = {}.freeze
 
     # Empty array to save on memory allocations
     EMPTY_ARRAY = [].freeze
 
-    private_constant :SUPPORTED_FLOW_ERRORS, :EMPTY_HASH, :EMPTY_ARRAY
+    private_constant(
+      :SUPPORTED_FLOW_ERRORS, :NON_RELOADABLE_FATAL_ERRORS, :EMPTY_HASH, :EMPTY_ARRAY
+    )
 
     def_delegators :config
 
@@ -57,6 +66,10 @@ module WaterDrop
       @default_variant = nil
       @client = nil
       @closing_thread_id = nil
+      @idempotent = nil
+      @transactional = nil
+      @idempotent_fatal_error_attempts = 0
+      @transaction_fatal_error_attempts = 0
 
       @status = Status.new
       @messages = []
@@ -74,13 +87,12 @@ module WaterDrop
     end
 
     # Sets up the whole configuration and initializes all that is needed
-    # @param block [Block] configuration block
-    def setup(&block)
+    def setup(...)
       raise Errors::ProducerAlreadyConfiguredError, id unless @status.initial?
 
       @config = Config
                 .new
-                .setup(&block)
+                .setup(...)
                 .config
 
       @id = @config.id
@@ -209,15 +221,6 @@ module WaterDrop
 
     alias variant with
 
-    # @return [Boolean] true if current producer is idempotent
-    def idempotent?
-      # Every transactional producer is idempotent by default always
-      return true if transactional?
-      return @idempotent if instance_variable_defined?(:'@idempotent')
-
-      @idempotent = config.kafka.to_h.fetch(:'enable.idempotence', false)
-    end
-
     # Returns and caches the middleware object that may be used
     # @return [WaterDrop::Producer::Middleware]
     def middleware
@@ -476,12 +479,46 @@ module WaterDrop
         message[:topic_config] = current_variant.topic_config
       end
 
-      if transactional?
-        transaction { client.produce(**message) }
-      else
-        client.produce(**message)
-      end
+      result = if transactional?
+                 transaction { client.produce(**message) }
+               else
+                 client.produce(**message)
+               end
+
+      # Reset attempts counter on successful produce
+      @idempotent_fatal_error_attempts = 0
+
+      result
     rescue SUPPORTED_FLOW_ERRORS.first => e
+      # Check if this is a fatal error on an idempotent producer and we should reload
+      if idempotent_reloadable?(e)
+        # Check if we've exceeded max reload attempts
+        raise unless idempotent_retryable?
+
+        # Increment attempts before reload
+        @idempotent_fatal_error_attempts += 1
+
+        # Instrument error.occurred before attempting reload for visibility
+        @monitor.instrument(
+          'error.occurred',
+          producer_id: id,
+          error: e,
+          type: 'librdkafka.idempotent_fatal_error',
+          attempt: @idempotent_fatal_error_attempts
+        )
+
+        # Attempt to reload the producer
+        idempotent_reload_client_on_fatal_error(@idempotent_fatal_error_attempts)
+
+        # Wait before retrying to avoid rapid reload loops
+        sleep(@config.wait_backoff_on_idempotent_fatal_error / 1_000.0)
+
+        # After reload, retry the produce operation
+        @operations_in_progress.decrement
+
+        retry
+      end
+
       # Unless we want to wait and retry and it's a full queue, we raise normally
       raise unless @config.wait_on_queue_full
       raise unless e.code == :queue_full
@@ -490,7 +527,7 @@ module WaterDrop
       # in an infinite loop, effectively hanging the processing
       raise unless monotonic_now - produce_time < @config.wait_timeout_on_queue_full
 
-      label = caller_locations(2, 1)[0].label.split(' ').last.split('#').last
+      label = caller_locations(2, 1)[0].label.split.last.split('#').last
 
       # We use this syntax here because we want to preserve the original `#cause` when we
       # instrument the error and there is no way to manually assign `#cause` value. We want to keep

data/lib/waterdrop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '2.8.11'
+  VERSION = '2.8.12'
 end

data/waterdrop.gemspec

@@ -7,7 +7,7 @@ require 'waterdrop/version'
 
 Gem::Specification.new do |spec|
   spec.name          = 'waterdrop'
-  spec.version       = ::WaterDrop::VERSION
+  spec.version       = WaterDrop::VERSION
   spec.platform      = Gem::Platform::RUBY
   spec.authors       = ['Maciej Mensfeld']
   spec.email         = %w[contact@karafka.io]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 2.8.11
+  version: 2.8.12
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -120,6 +120,7 @@ files:
 - lib/waterdrop/producer/buffer.rb
 - lib/waterdrop/producer/builder.rb
 - lib/waterdrop/producer/class_monitor.rb
+- lib/waterdrop/producer/idempotence.rb
 - lib/waterdrop/producer/status.rb
 - lib/waterdrop/producer/sync.rb
 - lib/waterdrop/producer/transactions.rb