waterdrop 2.7.2 → 2.7.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c580193c1e6a2bdcdcec6e95f99f4c6f7647a1d063871dde565282e9a2988c06
-  data.tar.gz: 7bc3f33ec9ce355682cea9047d62dfe8614f5319fc30e5a8b7cbd4d43d533182
+  metadata.gz: 83e99f22e3b3099cad3e459f171d7d43ab537fd62c50593997c3297846eb124a
+  data.tar.gz: 9583295a925fe82e6ca6a6899e0d37f644d0a30cc2a2359289d84ea78ab7e0ec
 SHA512:
-  metadata.gz: 8daa029fcc03d3170a1ffe5a17576e5906185391cd28482ecb6c2cf7ad56077e88e0818f87d92228b021dfd9eb22f3f314bce522a267431a7c60b98adf37a668
-  data.tar.gz: 9c408bc75a9225cf13078abc13b845d8b280f9342626f3a2caaf7216663f2a9da007224f47d29f6bb09a6c4600eb9e69e1965f6b86cb135e7255b1171d649d8a
+  metadata.gz: 017dd6f279e40d9c2d338e8ea2763fcab70e3ec99c0f229a1fb5c7037765d8f5a00c0dfe2d2ab932623b6b735ca27d9bd3ca3a8f012dd5aff33952f5eaa2b355
+  data.tar.gz: 6ff18fdd690e8e3f2341e2fa90a28b0972421559e5600c2c1c17b77d31d9882ed0476a3f2f7e6c33844f0b2a71aceb0b7f63bd8e6f8689084a41b6cfc09b6525

data/.github/workflows/ci.yml

@@ -18,6 +18,7 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
+          - '3.4.0-preview1'
           - '3.3'
           - '3.2'
           - '3.1'

data/.ruby-version

@@ -1 +1 @@
-3.3.1
+3.3.2

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 # WaterDrop changelog
 
+## 2.7.3 (2024-06-09)
+- [Enhancement] Introduce `reload_on_transaction_fatal_error` to reload the librdkafka after transactional failures
+- [Enhancement] Flush on fatal transactional errors.
+- [Enhancement] Add topic scope to `report_metric` (YadhuPrakash)
+- [Enhancement] Cache middleware reference saving 1 object allocation on each message dispatch.
+- [Enhancement] Provide `#idempotent?` similar to `#transactional?`.
+- [Enhancement] Provide alias to `#with` named `#variant`.
+- [Fix] Prevent from creating `acks` altering variants on idempotent producers.
+
 ## 2.7.2 (2024-05-09)
 - [Fix] Fix missing requirement of `delegate` for non-Rails use-cases. Always require delegate for variants usage (samsm)
 

data/Gemfile

@@ -12,6 +12,7 @@ end
 
 group :test do
   gem 'factory_bot'
+  gem 'ostruct'
   gem 'rspec'
   gem 'simplecov'
 end

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.7.2)
+    waterdrop (2.7.3)
       karafka-core (>= 2.4.0, < 3.0.0)
       karafka-rdkafka (>= 0.15.1)
       zeitwerk (~> 2.3)
@@ -9,7 +9,7 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (7.1.3.2)
+    activesupport (7.1.3.3)
       base64
       bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
@@ -22,7 +22,7 @@ GEM
     base64 (0.2.0)
     bigdecimal (3.1.8)
     byebug (11.1.3)
-    concurrent-ruby (1.2.3)
+    concurrent-ruby (1.3.1)
     connection_pool (2.4.1)
     diff-lcs (1.5.1)
     docile (1.4.0)
@@ -38,9 +38,10 @@ GEM
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    mini_portile2 (2.8.6)
-    minitest (5.22.3)
+    mini_portile2 (2.8.7)
+    minitest (5.23.1)
     mutex_m (0.2.0)
+    ostruct (0.6.0)
     rake (13.2.1)
     rspec (3.13.0)
       rspec-core (~> 3.13.0)
@@ -63,7 +64,7 @@ GEM
     simplecov_json_formatter (0.1.4)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    zeitwerk (2.6.13)
+    zeitwerk (2.6.15)
 
 PLATFORMS
   ruby
@@ -72,9 +73,10 @@ PLATFORMS
 DEPENDENCIES
   byebug
   factory_bot
+  ostruct
   rspec
   simplecov
   waterdrop!
 
 BUNDLED WITH
-   2.5.9
+   2.5.11

data/config/locales/errors.yml

@@ -17,6 +17,7 @@ en:
       wait_timeout_on_queue_full_format: must be a numeric that is equal or bigger to 0
       wait_backoff_on_transaction_command_format: must be a numeric that is equal or bigger to 0
       max_attempts_on_transaction_command_format: must be an integer that is equal or bigger than 1
+      reload_on_transaction_fatal_error_format: must be boolean
       oauth.token_provider_listener_format: 'must be false or respond to #on_oauthbearer_token_refresh'
 
     variant:
@@ -25,7 +26,7 @@ en:
       max_wait_timeout_format: must be an integer that is equal or bigger than 0
       kafka_key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
       kafka_key_not_per_topic: This config option cannot be set on a per topic basis
-      kafka_key_acks_not_changeable: Acks value cannot be redefined for a transactional producer
+      kafka_key_acks_not_changeable: Acks value cannot be redefined for a transactional or idempotent producer
 
     message:
       missing: must be present

data/lib/waterdrop/config.rb

@@ -72,6 +72,10 @@ module WaterDrop
     # option [Numeric] How many times to retry a retryable transaction related error before
     #   giving up
     setting :max_attempts_on_transaction_command, default: 5
+    # When a fatal transactional error occurs, should we close and recreate the underlying producer
+    # to keep going or should we stop. Since we will open a new instance and the failed transaction
+    # anyhow rolls back, we should be able to safely reload.
+    setting :reload_on_transaction_fatal_error, default: true
 
     # option [Boolean] should we send messages. Setting this to false can be really useful when
     #   testing and or developing because when set to false, won't actually ping Kafka but will

data/lib/waterdrop/contracts/config.rb

@@ -26,6 +26,7 @@ module WaterDrop
       required(:wait_timeout_on_queue_full) { |val| val.is_a?(Numeric) && val >= 0 }
       required(:wait_backoff_on_transaction_command) { |val| val.is_a?(Numeric) && val >= 0 }
       required(:max_attempts_on_transaction_command) { |val| val.is_a?(Integer) && val >= 1 }
+      required(:reload_on_transaction_fatal_error) { |val| [true, false].include?(val) }
 
       nested(:oauth) do
         required(:token_provider_listener) do |val|

data/lib/waterdrop/contracts/variant.rb

@@ -85,6 +85,23 @@ module WaterDrop
 
         errors
       end
+
+      # Prevent from creating variants altering acks when idempotent
+      virtual do |config, errors|
+        next true unless errors.empty?
+        # Relevant only for the transactional producer
+        next true unless config.fetch(:idempotent)
+
+        errors = []
+
+        config
+          .fetch(:topic_config)
+          .keys
+          .select { |key| key.to_s.include?('acks') }
+          .each { |key| errors << [[:kafka, key], :kafka_key_acks_not_changeable] }
+
+        errors
+      end
     end
   end
 end

data/lib/waterdrop/instrumentation/logger_listener.rb

@@ -129,6 +129,11 @@ module WaterDrop
         info(event, 'Closing producer')
       end
 
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_producer_reloaded(event)
+        info(event, 'Producer successfully reloaded')
+      end
+
       # @param event [Dry::Events::Event] event that happened with the error details
       def on_error_occurred(event)
         error = event[:error]

data/lib/waterdrop/instrumentation/notifications.rb

@@ -10,6 +10,7 @@ module WaterDrop
         producer.connected
         producer.closing
         producer.closed
+        producer.reloaded
 
         message.produced_async
         message.produced_sync

data/lib/waterdrop/instrumentation/vendors/datadog/metrics_listener.rb

@@ -199,6 +199,15 @@ module WaterDrop
                   tags: default_tags + ["broker:#{broker_statistics['nodename']}"]
                 )
               end
+            when :topics
+              statistics.fetch('topics').each_value do |topic_statistics|
+                public_send(
+                  metric.type,
+                  metric.name,
+                  topic_statistics.dig(*metric.key_location),
+                  tags: default_tags + ["topic:#{topic_statistics['topic']}"]
+                )
+              end
             else
               raise ArgumentError, metric.scope
             end

data/lib/waterdrop/producer/transactions.rb

@@ -4,10 +4,16 @@ module WaterDrop
   class Producer
     # Transactions related producer functionalities
     module Transactions
+      # We should never reload producer if it was fenced, otherwise we could end up with some sort
+      # of weird race-conditions
+      NON_RELOADABLE_ERRORS = %i[
+        fenced
+      ].freeze
+
       # Contract to validate that input for transactional offset storage is correct
       CONTRACT = Contracts::TransactionalOffset.new
 
-      private_constant :CONTRACT
+      private_constant :CONTRACT, :NON_RELOADABLE_ERRORS
 
       # Creates a transaction.
       #
@@ -79,11 +85,27 @@ module WaterDrop
           #
           # rubocop:disable Lint/RescueException
           rescue Exception => e
-            # rubocop:enable Lint/RescueException
-            with_transactional_error_handling(:abort) do
-              transactional_instrument(:aborted) { client.abort_transaction }
+            # This code is a bit tricky. We have an error and when it happens we try to rollback
+            # the transaction. However we may end up in a state where transaction aborting itself
+            # produces error. In such case we also want to handle it as fatal and reload client.
+            # This is why we catch this here
+            begin
+              # rubocop:enable Lint/RescueException
+              with_transactional_error_handling(:abort) do
+                transactional_instrument(:aborted) do
+                  client.abort_transaction
+                end
+              end
+            rescue StandardError => e
+              # If something from rdkafka leaks here, it means there was a non-retryable error that
+              # bubbled up. In such cases if we should, we do reload the underling client
+              transactional_reload_client_if_needed(e)
+
+              raise
             end
 
+            transactional_reload_client_if_needed(e)
+
             raise unless e.is_a?(WaterDrop::Errors::AbortTransaction)
           end
         end
@@ -197,7 +219,12 @@ module WaterDrop
           attempt: attempt
         )
 
-        raise if e.fatal?
+        if e.fatal?
+          # Reload the client on fatal errors if requested
+          transactional_reload_client_if_needed(e)
+
+          raise
+        end
 
         if do_retry
           # Backoff more and more before retries
@@ -212,12 +239,43 @@ module WaterDrop
           with_transactional_error_handling(:abort, allow_abortable: false) do
             transactional_instrument(:aborted) { client.abort_transaction }
           end
-
-          raise
         end
 
         raise
       end
+
+      # Reloads the underlying client instance if needed and allowed
+      #
+      # This should be used only in transactions as only then we can get fatal transactional
+      # errors and we can safely reload the client.
+      #
+      # @param error [Exception] any error that was raised
+      #
+      # @note We only reload on rdkafka errors that are a cause on messages dispatches.
+      # Because we reload on any errors where cause is `Rdkafka::RdkafkaError` (minus exclusions)
+      # this in theory can cause reload if it was something else that raised those in transactions,
+      # for example Karafka. This is a trade-off. Since any error anyhow will cause a rollback,
+      # putting aside performance implication of closing and reconnecting, this should not be an
+      # issue.
+      def transactional_reload_client_if_needed(error)
+        rd_error = error.is_a?(Rdkafka::RdkafkaError) ? error : error.cause
+
+        return unless rd_error.is_a?(Rdkafka::RdkafkaError)
+        return unless config.reload_on_transaction_fatal_error
+        return if NON_RELOADABLE_ERRORS.include?(rd_error.code)
+
+        @operating_mutex.synchronize do
+          @monitor.instrument(
+            'producer.reloaded',
+            producer_id: id
+          ) do
+            @client.flush(current_variant.max_wait_timeout)
+            purge
+            @client.close
+            @client = Builder.new.call(self, @config)
+          end
+        end
+      end
     end
   end
 end

data/lib/waterdrop/producer/variant.rb

@@ -98,7 +98,9 @@ module WaterDrop
           # We pass this to validation, to make sure no-one alters the `acks` value when operating
           # in the transactional mode as it causes librdkafka to crash ruby
           # @see https://github.com/confluentinc/librdkafka/issues/4710
-          transactional: @producer.transactional?
+          transactional: @producer.transactional?,
+          # We pass this for a similar reason as above
+          idempotent: @producer.idempotent?
         }
       end
     end

data/lib/waterdrop/producer.rb

@@ -21,7 +21,7 @@ module WaterDrop
 
     private_constant :SUPPORTED_FLOW_ERRORS, :EMPTY_HASH
 
-    def_delegators :config, :middleware
+    def_delegators :config
 
     # @return [String] uuid of the current producer
     attr_reader :id
@@ -143,6 +143,33 @@ module WaterDrop
       end
     end
 
+    # Builds the variant alteration and returns it.
+    #
+    # @param args [Object] anything `Producer::Variant` initializer accepts
+    # @return [WaterDrop::Producer::Variant] variant proxy to use with alterations
+    def with(**args)
+      ensure_active!
+
+      Variant.new(self, **args)
+    end
+
+    alias variant with
+
+    # @return [Boolean] true if current producer is idempotent
+    def idempotent?
+      # Every transactional producer is idempotent by default always
+      return true if transactional?
+      return @idempotent if instance_variable_defined?(:'@idempotent')
+
+      @idempotent = config.kafka.to_h.key?(:'enable.idempotence')
+    end
+
+    # Returns and caches the middleware object that may be used
+    # @return [WaterDrop::Producer::Middleware]
+    def middleware
+      @middleware ||= config.middleware
+    end
+
     # Flushes the buffers in a sync way and closes the producer
     # @param force [Boolean] should we force closing even with outstanding messages after the
     #   max wait timeout
@@ -210,16 +237,6 @@ module WaterDrop
       end
     end
 
-    # Builds the variant alteration and returns it.
-    #
-    # @param args [Object] anything `Producer::Variant` initializer accepts
-    # @return [WaterDrop::Producer::Variant] variant proxy to use with alterations
-    def with(**args)
-      ensure_active!
-
-      Variant.new(self, **args)
-    end
-
     # Closes the producer with forced close after timeout, purging any outgoing data
     def close!
       close(force: true)
@@ -306,7 +323,7 @@ module WaterDrop
       # in an infinite loop, effectively hanging the processing
       raise unless monotonic_now - produce_time < @config.wait_timeout_on_queue_full
 
-      label = caller_locations(2, 1)[0].label.split(' ').last
+      label = caller_locations(2, 1)[0].label.split(' ').last.split('#').last
 
       # We use this syntax here because we want to preserve the original `#cause` when we
       # instrument the error and there is no way to manually assign `#cause` value. We want to keep

data/lib/waterdrop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '2.7.2'
+  VERSION = '2.7.3'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 2.7.2
+  version: 2.7.3
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-05-09 00:00:00.000000000 Z
+date: 2024-06-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core