waterdrop 2.6.14 → 2.8.2

data/lib/waterdrop/producer/transactions.rb

@@ -4,10 +4,16 @@ module WaterDrop
   class Producer
     # Transactions related producer functionalities
     module Transactions
+      # We should never reload producer if it was fenced, otherwise we could end up with some sort
+      # of weird race-conditions
+      NON_RELOADABLE_ERRORS = %i[
+        fenced
+      ].freeze
+
       # Contract to validate that input for transactional offset storage is correct
       CONTRACT = Contracts::TransactionalOffset.new
 
-      private_constant :CONTRACT
+      private_constant :CONTRACT, :NON_RELOADABLE_ERRORS
 
       # Creates a transaction.
       #
@@ -29,7 +35,7 @@ module WaterDrop
       # with a transaction. One transaction per single dispatch and for `produce_many` it will be
       # a single transaction wrapping all messages dispatches (not one per message).
       #
-      # @return Block result
+      # @return Block result or `nil` in case of early break/return
       #
       # @example Simple transaction
       #   producer.transaction do
@@ -39,7 +45,7 @@ module WaterDrop
       # @example Aborted transaction - messages producer won't be visible by consumers
       #   producer.transaction do
       #     producer.produce_sync(topic: 'topic', payload: 'data')
-      #     throw(:abort)
+      #     raise WaterDrop::AbortTransaction
       #   end
       #
       # @example Use block result last handler to wait on all messages ack
@@ -49,26 +55,46 @@ module WaterDrop
       #
       #   handler.wait
       def transaction
+        unless transactional?
+          raise(
+            Errors::ProducerNotTransactionalError,
+            "#{id} is not transactional"
+          )
+        end
+
         # This will safely allow us to support one operation transactions so a transactional
         # producer can work without the transactional block if needed
         return yield if @transaction_mutex.owned?
 
         @transaction_mutex.synchronize do
+          ensure_active!
+
           transactional_instrument(:finished) do
             with_transactional_error_handling(:begin) do
               transactional_instrument(:started) { client.begin_transaction }
             end
 
             result = nil
-            commit = false
+            finished = false
 
-            catch(:abort) do
+            begin
               result = yield
-              commit = true
+              finished = true
+            rescue Exception => e
+              raise(e)
+            ensure
+              if !e && !finished
+                raise(
+                  Errors::EarlyTransactionExitNotAllowedError,
+                  <<~ERROR_MSG.tr("\n", ' ')
+                    Using `return`, `break` or `throw` to exit a transaction block is not allowed.
+                    If the `throw` came from `Timeout.timeout(duration)`, pass an exception class as
+                    a second argument so it doesn't use `throw` to abort its block.
+                  ERROR_MSG
+                )
+              end
             end
 
-            commit || raise(WaterDrop::Errors::AbortTransaction)
-
             with_transactional_error_handling(:commit) do
               transactional_instrument(:committed) { client.commit_transaction }
             end
@@ -76,19 +102,37 @@ module WaterDrop
             result
           # We need to handle any interrupt including critical in order not to have the transaction
           # running. This will also handle things like `IRB::Abort`
-          #
-          # rubocop:disable Lint/RescueException
           rescue Exception => e
-            # rubocop:enable Lint/RescueException
-            with_transactional_error_handling(:abort) do
-              transactional_instrument(:aborted) { client.abort_transaction }
+            # This code is a bit tricky. We have an error and when it happens we try to rollback
+            # the transaction. However we may end up in a state where transaction aborting itself
+            # produces error. In such case we also want to handle it as fatal and reload client.
+            # This is why we catch this here
+            begin
+              with_transactional_error_handling(:abort) do
+                transactional_instrument(:aborted) do
+                  client.abort_transaction
+                end
+              end
+            rescue StandardError => e
+              # If something from rdkafka leaks here, it means there was a non-retryable error that
+              # bubbled up. In such cases if we should, we do reload the underling client
+              transactional_reload_client_if_needed(e)
+
+              raise
             end
 
+            transactional_reload_client_if_needed(e)
+
             raise unless e.is_a?(WaterDrop::Errors::AbortTransaction)
           end
         end
       end
 
+      # @return [Boolean] true if we are in an active transaction
+      def transaction?
+        @transaction_mutex.owned?
+      end
+
       # @return [Boolean] Is this producer a transactional one
       def transactional?
         return @transactional if instance_variable_defined?(:'@transactional')
@@ -132,8 +176,7 @@ module WaterDrop
             client.send_offsets_to_transaction(
               consumer,
               tpl,
-              # This setting is at the moment in seconds and we require ms
-              @config.max_wait_timeout * 1_000
+              current_variant.max_wait_timeout
             )
           end
         end
@@ -193,11 +236,16 @@ module WaterDrop
           attempt: attempt
         )
 
-        raise if e.fatal?
+        if e.fatal?
+          # Reload the client on fatal errors if requested
+          transactional_reload_client_if_needed(e)
+
+          raise
+        end
 
         if do_retry
           # Backoff more and more before retries
-          sleep(config.wait_backoff_on_transaction_command * attempt)
+          sleep((config.wait_backoff_on_transaction_command / 1_000.0) * attempt)
 
           retry
         end
@@ -208,12 +256,43 @@ module WaterDrop
           with_transactional_error_handling(:abort, allow_abortable: false) do
             transactional_instrument(:aborted) { client.abort_transaction }
           end
-
-          raise
         end
 
         raise
       end
+
+      # Reloads the underlying client instance if needed and allowed
+      #
+      # This should be used only in transactions as only then we can get fatal transactional
+      # errors and we can safely reload the client.
+      #
+      # @param error [Exception] any error that was raised
+      #
+      # @note We only reload on rdkafka errors that are a cause on messages dispatches.
+      # Because we reload on any errors where cause is `Rdkafka::RdkafkaError` (minus exclusions)
+      # this in theory can cause reload if it was something else that raised those in transactions,
+      # for example Karafka. This is a trade-off. Since any error anyhow will cause a rollback,
+      # putting aside performance implication of closing and reconnecting, this should not be an
+      # issue.
+      def transactional_reload_client_if_needed(error)
+        rd_error = error.is_a?(Rdkafka::RdkafkaError) ? error : error.cause
+
+        return unless rd_error.is_a?(Rdkafka::RdkafkaError)
+        return unless config.reload_on_transaction_fatal_error
+        return if NON_RELOADABLE_ERRORS.include?(rd_error.code)
+
+        @operating_mutex.synchronize do
+          @monitor.instrument(
+            'producer.reloaded',
+            producer_id: id
+          ) do
+            @client.flush(current_variant.max_wait_timeout)
+            purge
+            @client.close
+            @client = Builder.new.call(self, @config)
+          end
+        end
+      end
     end
   end
 end

data/lib/waterdrop/producer/variant.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  class Producer
+    # Object that acts as a proxy allowing for alteration of certain low-level per-topic
+    # configuration and some other settings that users may find useful to alter, without having
+    # to create new producers with their underlying librdkafka instances.
+    #
+    # Since each librdkafka instance creates at least one TCP connection per broker, creating
+    # separate objects just to alter thing like `acks` may not be efficient and may lead to
+    # extensive usage of TCP connections, especially in bigger clusters.
+    #
+    # This variant object allows for "wrapping" of the producer with alteration of those settings
+    # in such a way, that two or more alterations can co-exist and share the same producer,
+    # effectively sharing the librdkafka client.
+    #
+    # Since this is an enhanced `SimpleDelegator` all `WaterDrop::Producer` APIs are preserved and
+    # a variant alteration can be used as a regular producer. The only important thing is to
+    # remember to only close it once.
+    #
+    # @note Not all settings are alterable. We only allow to alter things that are safe to be
+    #   altered as they have no impact on the producer. If there is a setting you consider
+    #   important and want to make it alterable, please open a GH issue for evaluation.
+    #
+    # @note Please be aware, that variant changes also affect buffers. If you overwrite the
+    #  `max_wait_timeout`, since buffers are shared (as they exist on producer level), flushing
+    #   may be impacted.
+    #
+    # @note `topic_config` is validated when created for the first time during message production.
+    #   This means, that configuration error may be raised only during dispatch. There is no
+    #   way out of this, since we need `librdkafka` instance to create the references.
+    class Variant < SimpleDelegator
+      # Empty hash we use as defaults for topic config.
+      # When rdkafka-ruby detects empty hash, it will use the librdkafka defaults
+      EMPTY_HASH = {}.freeze
+
+      private_constant :EMPTY_HASH
+
+      attr_reader :max_wait_timeout, :topic_config, :producer
+
+      # @param producer [WaterDrop::Producer] producer for which we want to have a variant
+      # @param max_wait_timeout [Integer, nil] alteration to max wait timeout or nil to use
+      #   default
+      # @param topic_config [Hash] extra topic configuration that can be altered.
+      # @param default [Boolean] is this a default variant or an altered one
+      # @see https://karafka.io/docs/Librdkafka-Configuration/#topic-configuration-properties
+      def initialize(
+        producer,
+        max_wait_timeout: producer.config.max_wait_timeout,
+        topic_config: EMPTY_HASH,
+        default: false
+      )
+        @producer = producer
+        @max_wait_timeout = max_wait_timeout
+        @topic_config = topic_config
+        @default = default
+        super(producer)
+
+        Contracts::Variant.new.validate!(to_h, Errors::VariantInvalidError)
+      end
+
+      # @return [Boolean] is this a default variant for this producer
+      def default?
+        @default
+      end
+
+      # We need to wrap any methods from our API that could use a variant alteration with the
+      # per thread variant injection. Since method_missing can be slow and problematic, it is just
+      # easier to use our public API components methods to ensure the variant is being injected.
+      [
+        Async,
+        Buffer,
+        Sync,
+        Transactions
+      ].each do |scope|
+        scope.instance_methods(false).each do |method_name|
+          class_eval <<-RUBY, __FILE__, __LINE__ + 1
+            def #{method_name}(*args, &block)
+              ref = Fiber.current.waterdrop_clients ||= {}
+              ref[@producer.id] = self
+
+              @producer.#{method_name}(*args, &block)
+            ensure
+              ref[@producer.id] = nil
+            end
+          RUBY
+        end
+      end
+
+      private
+
+      # @return [Hash] hash representation for contract validation to ensure basic sanity of the
+      #   settings.
+      def to_h
+        {
+          default: default?,
+          max_wait_timeout: max_wait_timeout,
+          topic_config: topic_config,
+          # We pass this to validation, to make sure no-one alters the `acks` value when operating
+          # in the transactional mode as it causes librdkafka to crash ruby
+          # @see https://github.com/confluentinc/librdkafka/issues/4710
+          transactional: @producer.transactional?,
+          # We pass this for a similar reason as above
+          idempotent: @producer.idempotent?
+        }
+      end
+    end
+  end
+end

data/lib/waterdrop/producer.rb

@@ -9,6 +9,10 @@ module WaterDrop
     include Buffer
     include Transactions
     include ::Karafka::Core::Helpers::Time
+    include ::Karafka::Core::Taggable
+
+    # Local storage for given thread waterdrop client references for variants
+    ::Fiber.send(:attr_accessor, :waterdrop_clients)
 
     # Which of the inline flow errors do we want to intercept and re-bind
     SUPPORTED_FLOW_ERRORS = [
@@ -16,12 +20,15 @@ module WaterDrop
       Rdkafka::Producer::DeliveryHandle::WaitTimeoutError
     ].freeze
 
-    # Empty has to save on memory allocations
+    # Empty hash to save on memory allocations
     EMPTY_HASH = {}.freeze
 
-    private_constant :SUPPORTED_FLOW_ERRORS, :EMPTY_HASH
+    # Empty array to save on memory allocations
+    EMPTY_ARRAY = [].freeze
+
+    private_constant :SUPPORTED_FLOW_ERRORS, :EMPTY_HASH, :EMPTY_ARRAY
 
-    def_delegators :config, :middleware
+    def_delegators :config
 
     # @return [String] uuid of the current producer
     attr_reader :id
@@ -65,6 +72,7 @@ module WaterDrop
       @id = @config.id
       @monitor = @config.monitor
       @contract = Contracts::Message.new(max_payload_size: @config.max_payload_size)
+      @default_variant = Variant.new(self, default: true)
       @status.configured!
     end
 
@@ -78,6 +86,7 @@ module WaterDrop
 
       # Don't allow to obtain a client reference for a producer that was not configured
       raise Errors::ProducerNotConfiguredError, id if @status.initial?
+      raise Errors::ProducerClosedError, id if @status.closed?
 
       @connecting_mutex.synchronize do
         return @client if @client && @pid == Process.pid
@@ -104,18 +113,6 @@ module WaterDrop
         @pid = Process.pid
         @client = Builder.new.call(self, @config)
 
-        # Register statistics runner for this particular type of callbacks
-        ::Karafka::Core::Instrumentation.statistics_callbacks.add(
-          @id,
-          Instrumentation::Callbacks::Statistics.new(@id, @client.name, @config.monitor)
-        )
-
-        # Register error tracking callback
-        ::Karafka::Core::Instrumentation.error_callbacks.add(
-          @id,
-          Instrumentation::Callbacks::Error.new(@id, @client.name, @config.monitor)
-        )
-
         @status.connected!
         @monitor.instrument('producer.connected', producer_id: id)
       end
@@ -145,73 +142,115 @@ module WaterDrop
           @messages = []
         end
 
-        @client.purge
+        # We should not purge if there is no client initialized
+        # It may not be initialized if we created a new producer that never connected to kafka,
+        # we used buffer and purged. In cases like this client won't exist
+        @connecting_mutex.synchronize do
+          @client&.purge
+        end
       end
     end
 
+    # Builds the variant alteration and returns it.
+    #
+    # @param args [Object] anything `Producer::Variant` initializer accepts
+    # @return [WaterDrop::Producer::Variant] variant proxy to use with alterations
+    def with(**args)
+      ensure_active!
+
+      Variant.new(self, **args)
+    end
+
+    alias variant with
+
+    # @return [Boolean] true if current producer is idempotent
+    def idempotent?
+      # Every transactional producer is idempotent by default always
+      return true if transactional?
+      return @idempotent if instance_variable_defined?(:'@idempotent')
+
+      @idempotent = config.kafka.to_h.key?(:'enable.idempotence')
+    end
+
+    # Returns and caches the middleware object that may be used
+    # @return [WaterDrop::Producer::Middleware]
+    def middleware
+      @middleware ||= config.middleware
+    end
+
     # Flushes the buffers in a sync way and closes the producer
     # @param force [Boolean] should we force closing even with outstanding messages after the
     #   max wait timeout
     def close(force: false)
-      @operating_mutex.synchronize do
-        return unless @status.active?
-
-        @monitor.instrument(
-          'producer.closed',
-          producer_id: id
-        ) do
-          @status.closing!
-          @monitor.instrument('producer.closing', producer_id: id)
-
-          # No need for auto-gc if everything got closed by us
-          # This should be used only in case a producer was not closed properly and forgotten
-          ObjectSpace.undefine_finalizer(id)
-
-          # We save this thread id because we need to bypass the activity verification on the
-          # producer for final flush of buffers.
-          @closing_thread_id = Thread.current.object_id
-
-          # Wait until all the outgoing operations are done. Only when no one is using the
-          # underlying client running operations we can close
-          sleep(0.001) until @operations_in_progress.value.zero?
-
-          # Flush has its own buffer mutex but even if it is blocked, flushing can still happen
-          # as we close the client after the flushing (even if blocked by the mutex)
-          flush(true)
-
-          # We should not close the client in several threads the same time
-          # It is safe to run it several times but not exactly the same moment
-          # We also mark it as closed only if it was connected, if not, it would trigger a new
-          # connection that anyhow would be immediately closed
-          if @client
-            # Why do we trigger it early instead of just having `#close` do it?
-            # The linger.ms time will be ignored for the duration of the call,
-            # queued messages will be sent to the broker as soon as possible.
-            begin
-              # `max_wait_timeout` is in seconds at the moment
-              @client.flush(@config.max_wait_timeout * 1_000) unless @client.closed?
-            # We can safely ignore timeouts here because any left outstanding requests
-            # will anyhow force wait on close if not forced.
-            # If forced, we will purge the queue and just close
-            rescue ::Rdkafka::RdkafkaError, Rdkafka::AbstractHandle::WaitTimeoutError
-              nil
-            ensure
-              # Purge fully the local queue in case of a forceful shutdown just to be sure, that
-              # there are no dangling messages. In case flush was successful, there should be
-              # none but we do it just in case it timed out
-              purge if force
+      # If we already own the transactional mutex, it means we are inside of a transaction and
+      # it should not we allowed to close the producer in such a case.
+      if @transaction_mutex.locked? && @transaction_mutex.owned?
+        raise Errors::ProducerTransactionalCloseAttemptError, id
+      end
+
+      # The transactional mutex here can be used even when no transactions are in use
+      # It prevents us from closing a mutex during transactions and is irrelevant in other cases
+      @transaction_mutex.synchronize do
+        @operating_mutex.synchronize do
+          return unless @status.active?
+
+          @monitor.instrument(
+            'producer.closed',
+            producer_id: id
+          ) do
+            @status.closing!
+            @monitor.instrument('producer.closing', producer_id: id)
+
+            # No need for auto-gc if everything got closed by us
+            # This should be used only in case a producer was not closed properly and forgotten
+            ObjectSpace.undefine_finalizer(id)
+
+            # We save this thread id because we need to bypass the activity verification on the
+            # producer for final flush of buffers.
+            @closing_thread_id = Thread.current.object_id
+
+            # Wait until all the outgoing operations are done. Only when no one is using the
+            # underlying client running operations we can close
+            sleep(0.001) until @operations_in_progress.value.zero?
+
+            # Flush has its own buffer mutex but even if it is blocked, flushing can still happen
+            # as we close the client after the flushing (even if blocked by the mutex)
+            flush(true)
+
+            # We should not close the client in several threads the same time
+            # It is safe to run it several times but not exactly the same moment
+            # We also mark it as closed only if it was connected, if not, it would trigger a new
+            # connection that anyhow would be immediately closed
+            if @client
+              # Why do we trigger it early instead of just having `#close` do it?
+              # The linger.ms time will be ignored for the duration of the call,
+              # queued messages will be sent to the broker as soon as possible.
+              begin
+                @client.flush(current_variant.max_wait_timeout) unless @client.closed?
+              # We can safely ignore timeouts here because any left outstanding requests
+              # will anyhow force wait on close if not forced.
+              # If forced, we will purge the queue and just close
+              rescue ::Rdkafka::RdkafkaError, Rdkafka::AbstractHandle::WaitTimeoutError
+                nil
+              ensure
+                # Purge fully the local queue in case of a forceful shutdown just to be sure, that
+                # there are no dangling messages. In case flush was successful, there should be
+                # none but we do it just in case it timed out
+                purge if force
+              end
+
+              @client.close
+
+              @client = nil
             end
 
-            @client.close
+            # Remove callbacks runners that were registered
+            ::Karafka::Core::Instrumentation.statistics_callbacks.delete(@id)
+            ::Karafka::Core::Instrumentation.error_callbacks.delete(@id)
+            ::Karafka::Core::Instrumentation.oauthbearer_token_refresh_callbacks.delete(@id)
 
-            @client = nil
+            @status.closed!
           end
-
-          # Remove callbacks runners that were registered
-          ::Karafka::Core::Instrumentation.statistics_callbacks.delete(@id)
-          ::Karafka::Core::Instrumentation.error_callbacks.delete(@id)
-
-          @status.closed!
         end
       end
     end
@@ -248,13 +287,23 @@ module WaterDrop
     # Waits on a given handler
     #
     # @param handler [Rdkafka::Producer::DeliveryHandle]
-    def wait(handler)
+    # @param raise_response_error [Boolean] should we raise the response error after we receive the
+    #   final result and it is an error.
+    def wait(handler, raise_response_error: true)
       handler.wait(
-        max_wait_timeout: @config.max_wait_timeout,
-        wait_timeout: @config.wait_timeout
+        # rdkafka max_wait_timeout is in seconds and we use ms
+        max_wait_timeout: current_variant.max_wait_timeout / 1_000.0,
+        raise_response_error: raise_response_error
       )
     end
 
+    # @return [Producer::Variant] the variant config. Either custom if built using `#with` or
+    #   a default one.
+    def current_variant
+      Fiber.current.waterdrop_clients ||= {}
+      Fiber.current.waterdrop_clients[id] || @default_variant
+    end
+
     # Runs the client produce method with a given message
     #
     # @param message [Hash] message we want to send
@@ -270,9 +319,17 @@ module WaterDrop
         ensure_active!
       end
 
-      # In case someone defines topic as a symbol, we need to convert it into a string as
-      # librdkafka does not accept symbols
-      message = message.merge(topic: message[:topic].to_s) if message[:topic].is_a?(Symbol)
+      # We basically only duplicate the message hash only if it is needed.
+      # It is needed when user is using a custom settings variant or when symbol is provided as
+      # the topic name. We should never mutate user input message as it may be a hash that the
+      # user is using for some other operations
+      if message[:topic].is_a?(Symbol) || !current_variant.default?
+        message = message.dup
+        # In case someone defines topic as a symbol, we need to convert it into a string as
+        # librdkafka does not accept symbols
+        message[:topic] = message[:topic].to_s
+        message[:topic_config] = current_variant.topic_config
+      end
 
       if transactional?
         transaction { client.produce(**message) }
@@ -286,9 +343,9 @@ module WaterDrop
       # If we're running for longer than the timeout, we need to re-raise the queue full.
       # This will prevent from situation where cluster is down forever and we just retry and retry
       # in an infinite loop, effectively hanging the processing
-      raise unless monotonic_now - produce_time < @config.wait_timeout_on_queue_full * 1_000
+      raise unless monotonic_now - produce_time < @config.wait_timeout_on_queue_full
 
-      label = caller_locations(2, 1)[0].label.split(' ').last
+      label = caller_locations(2, 1)[0].label.split(' ').last.split('#').last
 
       # We use this syntax here because we want to preserve the original `#cause` when we
       # instrument the error and there is no way to manually assign `#cause` value. We want to keep
@@ -297,22 +354,28 @@ module WaterDrop
       begin
         raise Errors::ProduceError, e.inspect
       rescue Errors::ProduceError => e
-        # We want to instrument on this event even when we restart it.
-        # The reason is simple: instrumentation and visibility.
-        # We can recover from this, but despite that we should be able to instrument this.
-        # If this type of event happens too often, it may indicate that the buffer settings are not
-        # well configured.
-        @monitor.instrument(
-          'error.occurred',
-          producer_id: id,
-          message: message,
-          error: e,
-          type: "message.#{label}"
-        )
+        # Users can configure this because in pipe-like flows with high throughput, queue full with
+        # retry may be used as a throttling system that will backoff and wait.
+        # In such scenarios this error notification can be removed and until queue full is
+        # retryable, it will not be raised as an error.
+        if @config.instrument_on_wait_queue_full
+          # We want to instrument on this event even when we restart it.
+          # The reason is simple: instrumentation and visibility.
+          # We can recover from this, but despite that we should be able to instrument this.
+          # If this type of event happens too often, it may indicate that the buffer settings are
+          # not well configured.
+          @monitor.instrument(
+            'error.occurred',
+            producer_id: id,
+            message: message,
+            error: e,
+            type: "message.#{label}"
+          )
+        end
 
         # We do not poll the producer because polling happens in a background thread
         # It also should not be a frequent case (queue full), hence it's ok to just throttle.
-        sleep @config.wait_backoff_on_queue_full
+        sleep @config.wait_backoff_on_queue_full / 1_000.0
       end
 
       @operations_in_progress.decrement

data/lib/waterdrop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '2.6.14'
+  VERSION = '2.8.2'
 end

data/lib/waterdrop.rb

@@ -3,6 +3,7 @@
 # External components
 # delegate should be removed because we don't need it, we just add it because of ruby-kafka
 %w[
+  delegate
   forwardable
   json
   zeitwerk