waterdrop 2.6.14 → 2.8.2

data/docker-compose.yml

@@ -1,9 +1,7 @@
-version: '2'
-
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.5.3
+    image: confluentinc/cp-kafka:7.8.1
 
     ports:
       - 9092:9092

data/lib/waterdrop/clients/rdkafka.rb

@@ -11,20 +11,50 @@ module WaterDrop
         # @param producer [WaterDrop::Producer] producer instance with its config, etc
         # @note We overwrite this that way, because we do not care
         def new(producer)
-          config = producer.config.kafka.to_h
+          kafka_config = producer.config.kafka.to_h
+          monitor = producer.config.monitor
 
-          client = ::Rdkafka::Config.new(config).producer
+          client = ::Rdkafka::Config.new(kafka_config).producer(native_kafka_auto_start: false)
+
+          # Register statistics runner for this particular type of callbacks
+          ::Karafka::Core::Instrumentation.statistics_callbacks.add(
+            producer.id,
+            Instrumentation::Callbacks::Statistics.new(producer.id, client.name, monitor)
+          )
+
+          # Register error tracking callback
+          ::Karafka::Core::Instrumentation.error_callbacks.add(
+            producer.id,
+            Instrumentation::Callbacks::Error.new(producer.id, client.name, monitor)
+          )
+
+          # Register oauth bearer refresh for this particular type of callbacks
+          ::Karafka::Core::Instrumentation.oauthbearer_token_refresh_callbacks.add(
+            producer.id,
+            Instrumentation::Callbacks::OauthbearerTokenRefresh.new(client, monitor)
+          )
 
           # This callback is not global and is per client, thus we do not have to wrap it with a
           # callbacks manager to make it work
           client.delivery_callback = Instrumentation::Callbacks::Delivery.new(
             producer.id,
             producer.transactional?,
-            producer.config.monitor
+            monitor
           )
 
+          oauth_listener = producer.config.oauth.token_provider_listener
+          # We need to subscribe the oauth listener here because we want it to be ready before
+          # any producer callbacks run. In theory because WaterDrop rdkafka producer is lazy loaded
+          # we would have enough time to make user subscribe it himself, but then it would not
+          # coop with auto-configuration coming from Karafka. The way it is done below, if it is
+          # configured it will be subscribed and if not, user always can subscribe it himself as
+          # long as it is done prior to first usage
+          monitor.subscribe(oauth_listener) if oauth_listener
+
+          client.start
+
           # Switch to the transactional mode if user provided the transactional id
-          client.init_transactions if config.key?(:'transactional.id')
+          client.init_transactions if kafka_config.key?(:'transactional.id')
 
           client
         end

data/lib/waterdrop/config.rb

@@ -12,7 +12,12 @@ module WaterDrop
       'client.id': 'waterdrop',
       # emit librdkafka statistics every five seconds. This is used in instrumentation.
       # When disabled, part of metrics will not be published and available.
-      'statistics.interval.ms': 5_000
+      'statistics.interval.ms': 5_000,
+      # We set it to a value that is lower than `max_wait_timeout` to have a final verdict upon
+      # sync delivery
+      'message.timeout.ms': 50_000,
+      # Must be more or equal to `message.timeout.ms` defaults
+      'transaction.timeout.ms': 55_000
     }.freeze
 
     private_constant :KAFKA_DEFAULTS
@@ -25,7 +30,7 @@ module WaterDrop
     setting(
       :id,
       default: false,
-      constructor: ->(id) { id || SecureRandom.hex(6) }
+      constructor: ->(id) { id || "waterdrop-#{SecureRandom.hex(6)}" }
     )
     # option [Instance] logger that we want to use
     # @note Due to how rdkafka works, this setting is global for all the producers
@@ -44,12 +49,8 @@ module WaterDrop
     # option [Integer] max payload size allowed for delivery to Kafka
     setting :max_payload_size, default: 1_000_012
     # option [Integer] Wait that long for the delivery report or raise an error if this takes
-    #   longer than the timeout.
-    setting :max_wait_timeout, default: 5
-    # option [Numeric] how long should we wait between re-checks on the availability of the
-    #   delivery report. In a really robust systems, this describes the min-delivery time
-    #   for a single sync message when produced in isolation
-    setting :wait_timeout, default: 0.005 # 5 milliseconds
+    #   longer than the timeout ms.
+    setting :max_wait_timeout, default: 60_000
     # option [Boolean] should we upon detecting full librdkafka queue backoff and retry or should
     #   we raise an exception.
     #   When this is set to `true`, upon full queue, we won't raise an error. There will be error
@@ -60,15 +61,21 @@ module WaterDrop
     # option [Integer] how long (in seconds) should we backoff before a retry when queue is full
     #   The retry will happen with the same message and backoff should give us some time to
     #   dispatch previously buffered messages.
-    setting :wait_backoff_on_queue_full, default: 0.1
-    # option [Numeric] how many seconds should we wait with the backoff on queue having space for
+    setting :wait_backoff_on_queue_full, default: 100
+    # option [Numeric] how many ms should we wait with the backoff on queue having space for
     # more messages before re-raising the error.
-    setting :wait_timeout_on_queue_full, default: 10
+    setting :wait_timeout_on_queue_full, default: 10_000
+    # option [Boolean] should we instrument non-critical, retryable queue full errors
+    setting :instrument_on_wait_queue_full, default: true
     # option [Numeric] How long to wait before retrying a retryable transaction related error
-    setting :wait_backoff_on_transaction_command, default: 0.5
+    setting :wait_backoff_on_transaction_command, default: 500
     # option [Numeric] How many times to retry a retryable transaction related error before
     #   giving up
     setting :max_attempts_on_transaction_command, default: 5
+    # When a fatal transactional error occurs, should we close and recreate the underlying producer
+    # to keep going or should we stop. Since we will open a new instance and the failed transaction
+    # anyhow rolls back, we should be able to safely reload.
+    setting :reload_on_transaction_fatal_error, default: true
 
     # option [Boolean] should we send messages. Setting this to false can be really useful when
     #   testing and or developing because when set to false, won't actually ping Kafka but will
@@ -86,6 +93,14 @@ module WaterDrop
       constructor: ->(middleware) { middleware || WaterDrop::Middleware.new }
     )
 
+    # Namespace for oauth related configuration
+    setting :oauth do
+      # option [false, #call] Listener for using oauth bearer. This listener will be able to
+      #   get the client name to decide whether to use a single multi-client token refreshing
+      #   or have separate tokens per instance.
+      setting :token_provider_listener, default: false
+    end
+
     # Configuration method
     # @yield Runs a block of code providing a config singleton instance to it
     # @yieldparam [WaterDrop::Config] WaterDrop config instance

data/lib/waterdrop/contracts/config.rb

@@ -14,14 +14,25 @@ module WaterDrop
 
       required(:id) { |val| val.is_a?(String) && !val.empty? }
       required(:logger) { |val| !val.nil? }
+      required(:monitor) { |val| !val.nil? }
       required(:deliver) { |val| [true, false].include?(val) }
       required(:max_payload_size) { |val| val.is_a?(Integer) && val >= 1 }
       required(:max_wait_timeout) { |val| val.is_a?(Numeric) && val >= 0 }
-      required(:wait_timeout) { |val| val.is_a?(Numeric) && val.positive? }
+      required(:client_class) { |val| !val.nil? }
       required(:kafka) { |val| val.is_a?(Hash) && !val.empty? }
       required(:wait_on_queue_full) { |val| [true, false].include?(val) }
+      required(:instrument_on_wait_queue_full) { |val| [true, false].include?(val) }
       required(:wait_backoff_on_queue_full) { |val| val.is_a?(Numeric) && val >= 0 }
       required(:wait_timeout_on_queue_full) { |val| val.is_a?(Numeric) && val >= 0 }
+      required(:wait_backoff_on_transaction_command) { |val| val.is_a?(Numeric) && val >= 0 }
+      required(:max_attempts_on_transaction_command) { |val| val.is_a?(Integer) && val >= 1 }
+      required(:reload_on_transaction_fatal_error) { |val| [true, false].include?(val) }
+
+      nested(:oauth) do
+        required(:token_provider_listener) do |val|
+          val == false || val.respond_to?(:on_oauthbearer_token_refresh)
+        end
+      end
 
       # rdkafka allows both symbols and strings as keys for config but then casts them to strings
       # This can be confusing, so we expect all keys to be symbolized

data/lib/waterdrop/contracts/variant.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Contracts
+    # Variant validator to ensure basic sanity of the variant alteration data
+    class Variant < ::Karafka::Core::Contractable::Contract
+      # Taken from librdkafka config
+      # Those values can be changed on a per topic basis. We do not support experimental or
+      # deprecated values. We also do not support settings that would break rdkafka-ruby
+      #
+      # @see https://karafka.io/docs/Librdkafka-Configuration/#topic-configuration-properties
+      TOPIC_CONFIG_KEYS = %i[
+        acks
+        compression.codec
+        compression.level
+        compression.type
+        delivery.timeout.ms
+        message.timeout.ms
+        partitioner
+        request.required.acks
+        request.timeout.ms
+      ].freeze
+
+      # Boolean values
+      BOOLEANS = [true, false].freeze
+
+      private_constant :TOPIC_CONFIG_KEYS, :BOOLEANS
+
+      configure do |config|
+        config.error_messages = YAML.safe_load(
+          File.read(
+            File.join(WaterDrop.gem_root, 'config', 'locales', 'errors.yml')
+          )
+        ).fetch('en').fetch('validations').fetch('variant')
+      end
+
+      required(:default) { |val| BOOLEANS.include?(val) }
+      required(:max_wait_timeout) { |val| val.is_a?(Numeric) && val >= 0 }
+
+      # Checks if all keys are symbols
+      virtual do |config, errors|
+        next true unless errors.empty?
+
+        errors = []
+
+        config
+          .fetch(:topic_config)
+          .keys
+          .reject { |key| key.is_a?(Symbol) }
+          .each { |key| errors << [[:kafka, key], :kafka_key_must_be_a_symbol] }
+
+        errors
+      end
+
+      # Checks if we have any keys that are not allowed
+      virtual do |config, errors|
+        next true unless errors.empty?
+
+        errors = []
+
+        config
+          .fetch(:topic_config)
+          .keys
+          .reject { |key| TOPIC_CONFIG_KEYS.include?(key) }
+          .each { |key| errors << [[:kafka, key], :kafka_key_not_per_topic] }
+
+        errors
+      end
+
+      # Ensure, that acks is not changed when in transactional mode
+      # acks needs to be set to 'all' and should not be changed when working with transactional
+      # producer as it causes librdkafka to crash
+      virtual do |config, errors|
+        next true unless errors.empty?
+        # Relevant only for the transactional producer
+        next true unless config.fetch(:transactional)
+
+        errors = []
+
+        config
+          .fetch(:topic_config)
+          .keys
+          .select { |key| key.to_s.include?('acks') }
+          .each { |key| errors << [[:kafka, key], :kafka_key_acks_not_changeable] }
+
+        errors
+      end
+
+      # Prevent from creating variants altering acks when idempotent
+      virtual do |config, errors|
+        next true unless errors.empty?
+        # Relevant only for the transactional producer
+        next true unless config.fetch(:idempotent)
+
+        errors = []
+
+        config
+          .fetch(:topic_config)
+          .keys
+          .select { |key| key.to_s.include?('acks') }
+          .each { |key| errors << [[:kafka, key], :kafka_key_acks_not_changeable] }
+
+        errors
+      end
+    end
+  end
+end

data/lib/waterdrop/errors.rb

@@ -9,6 +9,9 @@ module WaterDrop
     # Raised when configuration doesn't match with validation contract
     ConfigurationInvalidError = Class.new(BaseError)
 
+    # Raised when variant alteration is not valid
+    VariantInvalidError = Class.new(BaseError)
+
     # Raised when we want to use a producer that was not configured
     ProducerNotConfiguredError = Class.new(BaseError)
 
@@ -22,12 +25,18 @@ module WaterDrop
     # Raised when there was an attempt to use a closed producer
     ProducerClosedError = Class.new(BaseError)
 
+    # Raised if you attempt to close the producer from within a transaction. This is not allowed.
+    ProducerTransactionalCloseAttemptError = Class.new(BaseError)
+
     # Raised when we want to send a message that is invalid (impossible topic, etc)
     MessageInvalidError = Class.new(BaseError)
 
     # Raised when we want to commit transactional offset and the input is invalid
     TransactionalOffsetInvalidError = Class.new(BaseError)
 
+    # Raised when transaction attempt happens on a non-transactional producer
+    ProducerNotTransactionalError = Class.new(BaseError)
+
     # Raised when we've got an unexpected status. This should never happen. If it does, please
     # contact us as it is an error.
     StatusInvalidError = Class.new(BaseError)
@@ -44,6 +53,9 @@ module WaterDrop
     # graceful transaction aborting
     AbortTransaction = Class.new(BaseError)
 
+    # Do not use `break`, `return` or `throw` inside of the transaction blocks
+    EarlyTransactionExitNotAllowedError = Class.new(BaseError)
+
     # Raised when during messages producing something bad happened inline
     class ProduceManyError < ProduceError
       attr_reader :dispatched
@@ -57,4 +69,8 @@ module WaterDrop
       end
     end
   end
+
+  # Alias so we can have a nicer API to abort transactions
+  # This makes referencing easier
+  AbortTransaction = Errors::AbortTransaction
 end

data/lib/waterdrop/instrumentation/callbacks/delivery.rb

@@ -44,6 +44,17 @@ module WaterDrop
           else
             instrument_error(delivery_report)
           end
+        # This runs from the rdkafka thread, thus we want to safe-guard it and prevent absolute
+        # crashes even if the instrumentation code fails. If it would bubble-up, it could crash
+        # the rdkafka background thread
+        rescue StandardError => e
+          @monitor.instrument(
+            'error.occurred',
+            caller: self,
+            error: e,
+            producer_id: @producer_id,
+            type: 'callbacks.delivery.error'
+          )
         end
 
         private

data/lib/waterdrop/instrumentation/callbacks/error.rb

@@ -32,6 +32,17 @@ module WaterDrop
             producer_id: @producer_id,
             type: 'librdkafka.error'
           )
+        # This runs from the rdkafka thread, thus we want to safe-guard it and prevent absolute
+        # crashes even if the instrumentation code fails. If it would bubble-up, it could crash
+        # the rdkafka background thread
+        rescue StandardError => e
+          @monitor.instrument(
+            'error.occurred',
+            caller: self,
+            error: e,
+            producer_id: @producer_id,
+            type: 'callbacks.error.error'
+          )
         end
       end
     end

data/lib/waterdrop/instrumentation/callbacks/oauthbearer_token_refresh.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Instrumentation
+    module Callbacks
+      # Callback that is triggered when oauth token needs to be refreshed.
+      class OauthbearerTokenRefresh
+        # @param bearer [Rdkafka::Producer] given rdkafka instance. It is needed as
+        #   we need to have a reference to call `#oauthbearer_set_token` or
+        #   `#oauthbearer_set_token_failure` upon the event.
+        # @param monitor [WaterDrop::Instrumentation::Monitor] monitor we are using
+        def initialize(bearer, monitor)
+          @bearer = bearer
+          @monitor = monitor
+        end
+
+        # Upon receiving of this event, user is required to invoke either `#oauthbearer_set_token`
+        # or `#oauthbearer_set_token_failure` on the `event[:bearer]` depending whether token
+        # obtaining was successful or not.
+        #
+        # Please refer to WaterDrop and Karafka documentation or `Rdkafka::Helpers::OAuth`
+        # documentation directly for exact parameters of those methods.
+        #
+        # @param _rd_config [Rdkafka::Config]
+        # @param bearer_name [String] name of the bearer for which we refresh
+        def call(_rd_config, bearer_name)
+          return unless @bearer.name == bearer_name
+
+          @monitor.instrument(
+            'oauthbearer.token_refresh',
+            bearer: @bearer,
+            caller: self
+          )
+        # This runs from the rdkafka thread, thus we want to safe-guard it and prevent absolute
+        # crashes even if the instrumentation code fails. If it would bubble-up, it could crash
+        # the rdkafka background thread
+        rescue StandardError => e
+          @monitor.instrument(
+            'error.occurred',
+            caller: self,
+            error: e,
+            producer_id: @producer_id,
+            type: 'callbacks.oauthbearer_token_refresh.error'
+          )
+        end
+      end
+    end
+  end
+end

data/lib/waterdrop/instrumentation/callbacks/statistics.rb

@@ -34,6 +34,17 @@ module WaterDrop
             producer_id: @producer_id,
             statistics: @statistics_decorator.call(statistics)
           )
+        # This runs from the rdkafka thread, thus we want to safe-guard it and prevent absolute
+        # crashes even if the instrumentation code fails. If it would bubble-up, it could crash
+        # the rdkafka background thread
+        rescue StandardError => e
+          @monitor.instrument(
+            'error.occurred',
+            caller: self,
+            error: e,
+            producer_id: @producer_id,
+            type: 'callbacks.statistics.error'
+          )
         end
       end
     end

data/lib/waterdrop/instrumentation/logger_listener.rb

@@ -26,7 +26,7 @@ module WaterDrop
       def on_message_produced_async(event)
         message = event[:message]
 
-        info(event, "Async producing of a message to '#{message[:topic]}' topic")
+        info(event, "Message to '#{message[:topic]}' topic was delegated to a dispatch queue")
 
         return unless log_messages?
 
@@ -49,7 +49,10 @@ module WaterDrop
         messages = event[:messages]
         topics_count = messages.map { |message| "'#{message[:topic]}'" }.uniq.count
 
-        info(event, "Async producing of #{messages.size} messages to #{topics_count} topics")
+        info(
+          event,
+          "#{messages.size} messages to #{topics_count} topics were delegated to a dispatch queue"
+        )
 
         return unless log_messages?
 
@@ -129,6 +132,11 @@ module WaterDrop
         info(event, 'Closing producer')
       end
 
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_producer_reloaded(event)
+        info(event, 'Producer successfully reloaded')
+      end
+
       # @param event [Dry::Events::Event] event that happened with the error details
       def on_error_occurred(event)
         error = event[:error]
@@ -188,7 +196,7 @@ module WaterDrop
       # @param log_message [String] message we want to publish
       def info(event, log_message)
         if event.payload.key?(:time)
-          @logger.info("[#{event[:producer_id]}] #{log_message} took #{event[:time]} ms")
+          @logger.info("[#{event[:producer_id]}] #{log_message} took #{event[:time].round(2)} ms")
         else
           @logger.info("[#{event[:producer_id]}] #{log_message}")
         end

data/lib/waterdrop/instrumentation/notifications.rb

@@ -10,6 +10,7 @@ module WaterDrop
         producer.connected
         producer.closing
         producer.closed
+        producer.reloaded
 
         message.produced_async
         message.produced_sync
@@ -21,6 +22,8 @@ module WaterDrop
         messages.produced_sync
         messages.buffered
 
+        oauthbearer.token_refresh
+
         transaction.started
         transaction.committed
         transaction.aborted

data/lib/waterdrop/instrumentation/vendors/datadog/metrics_listener.rb

@@ -36,7 +36,7 @@ module WaterDrop
           setting :rd_kafka_metrics, default: [
             # Client metrics
             RdKafkaMetric.new(:count, :root, 'calls', 'tx_d'),
-            RdKafkaMetric.new(:histogram, :root, 'queue.size', 'msg_cnt_d'),
+            RdKafkaMetric.new(:histogram, :root, 'queue.size', 'msg_cnt'),
 
             # Broker metrics
             RdKafkaMetric.new(:count, :brokers, 'deliver.attempts', 'txretries_d'),
@@ -199,6 +199,15 @@ module WaterDrop
                   tags: default_tags + ["broker:#{broker_statistics['nodename']}"]
                 )
               end
+            when :topics
+              statistics.fetch('topics').each_value do |topic_statistics|
+                public_send(
+                  metric.type,
+                  metric.name,
+                  topic_statistics.dig(*metric.key_location),
+                  tags: default_tags + ["topic:#{topic_statistics['topic']}"]
+                )
+              end
             else
               raise ArgumentError, metric.scope
             end

data/lib/waterdrop/producer/buffer.rb

@@ -12,9 +12,6 @@ module WaterDrop
       def buffer(message)
         ensure_active!
 
-        message = middleware.run(message)
-        validate_message!(message)
-
         @monitor.instrument(
           'message.buffered',
           producer_id: id,
@@ -32,9 +29,6 @@ module WaterDrop
       def buffer_many(messages)
         ensure_active!
 
-        messages = middleware.run_many(messages)
-        messages.each { |message| validate_message!(message) }
-
         @monitor.instrument(
           'messages.buffered',
           producer_id: id,

data/lib/waterdrop/producer/sync.rb

@@ -52,8 +52,8 @@ module WaterDrop
       # @return [Array<Rdkafka::Producer::DeliveryReport>] delivery reports
       #
       # @raise [Rdkafka::RdkafkaError] When adding the messages to rdkafka's queue failed
-      # @raise [Rdkafka::Producer::WaitTimeoutError] When the timeout has been reached and the
-      #   some handles are still pending
+      # @raise [Rdkafka::Producer::WaitTimeoutError] When the timeout has been reached and some
+      #   handles are still pending
       # @raise [Errors::MessageInvalidError] When any of the provided messages details are invalid
       #   and the message could not be sent to Kafka
       def produce_many_sync(messages)
@@ -61,17 +61,37 @@ module WaterDrop
         messages.each { |message| validate_message!(message) }
 
         dispatched = []
+        inline_error = nil
 
         @monitor.instrument('messages.produced_sync', producer_id: id, messages: messages) do
-          with_transaction_if_transactional do
-            messages.each do |message|
-              dispatched << produce(message)
+          # While most of the librdkafka errors are async and not inline, there are some like
+          # buffer overflow that can leak in during the `#produce` itself. When this happens, we
+          # still (since it's a sync mode) need to wait on deliveries of things that were
+          # successfully delegated to librdkafka. To do this, we catch the inline error and we
+          # wait on messages that were in the buffer to reach final state. Then if no error, we
+          # check each for error and if none all good. If there was an inline error, we re-raise
+          # it with the handlers in final states.
+          #
+          # Such flow ensures, that we never end up with handlers not being in the final states
+          # for the sync flow
+          begin
+            with_transaction_if_transactional do
+              messages.each do |message|
+                dispatched << produce(message)
+              end
             end
+          rescue *SUPPORTED_FLOW_ERRORS => e
+            inline_error = e
           end
 
-          dispatched.map! do |handler|
-            wait(handler)
-          end
+          # This will ensure, that we have all verdicts before raising the failure, so we pass
+          # all delivery handles having a final verdict
+          dispatched.each { |handler| wait(handler, raise_response_error: false) }
+
+          raise(inline_error) if inline_error
+
+          # This will raise an error on the first error that have happened
+          dispatched.each { |handler| wait(handler) }
 
           dispatched
         end
@@ -82,7 +102,11 @@ module WaterDrop
           'error.occurred',
           producer_id: id,
           messages: messages,
-          dispatched: dispatched,
+          # If it is a transactional producer nothing was successfully dispatched on error, thus
+          # we never return any dispatched handlers. While those messages might have reached
+          # Kafka, in transactional mode they will not be visible to consumers with correct
+          # isolation level.
+          dispatched: transactional? ? EMPTY_ARRAY : dispatched,
           error: re_raised,
           type: 'messages.produce_many_sync'
         )