waterdrop 2.0.7 → 2.6.14

data/lib/{water_drop → waterdrop}/config.rb

@@ -5,11 +5,14 @@
 module WaterDrop
   # Configuration object for setting up all options required by WaterDrop
   class Config
-    include Dry::Configurable
+    include ::Karafka::Core::Configurable
 
     # Defaults for kafka settings, that will be overwritten only if not present already
     KAFKA_DEFAULTS = {
-      'client.id' => 'waterdrop'
+      'client.id': 'waterdrop',
+      # emit librdkafka statistics every five seconds. This is used in instrumentation.
+      # When disabled, part of metrics will not be published and available.
+      'statistics.interval.ms': 5_000
     }.freeze
 
     private_constant :KAFKA_DEFAULTS
@@ -22,7 +25,7 @@ module WaterDrop
     setting(
       :id,
       default: false,
-      constructor: ->(id) { id || SecureRandom.uuid }
+      constructor: ->(id) { id || SecureRandom.hex(6) }
     )
     # option [Instance] logger that we want to use
     # @note Due to how rdkafka works, this setting is global for all the producers
@@ -47,13 +50,41 @@ module WaterDrop
     #   delivery report. In a really robust systems, this describes the min-delivery time
     #   for a single sync message when produced in isolation
     setting :wait_timeout, default: 0.005 # 5 milliseconds
+    # option [Boolean] should we upon detecting full librdkafka queue backoff and retry or should
+    #   we raise an exception.
+    #   When this is set to `true`, upon full queue, we won't raise an error. There will be error
+    #   in the `error.occurred` notification pipeline with a proper type as while this is
+    #   recoverable, in a high number it still may mean issues.
+    #   Waiting is one of the recommended strategies.
+    setting :wait_on_queue_full, default: true
+    # option [Integer] how long (in seconds) should we backoff before a retry when queue is full
+    #   The retry will happen with the same message and backoff should give us some time to
+    #   dispatch previously buffered messages.
+    setting :wait_backoff_on_queue_full, default: 0.1
+    # option [Numeric] how many seconds should we wait with the backoff on queue having space for
+    # more messages before re-raising the error.
+    setting :wait_timeout_on_queue_full, default: 10
+    # option [Numeric] How long to wait before retrying a retryable transaction related error
+    setting :wait_backoff_on_transaction_command, default: 0.5
+    # option [Numeric] How many times to retry a retryable transaction related error before
+    #   giving up
+    setting :max_attempts_on_transaction_command, default: 5
+
     # option [Boolean] should we send messages. Setting this to false can be really useful when
     #   testing and or developing because when set to false, won't actually ping Kafka but will
     #   run all the validations, etc
     setting :deliver, default: true
+    # option [Class] class for usage when creating the underlying client used to dispatch messages
+    setting :client_class, default: Clients::Rdkafka
     # rdkafka options
     # @see https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
     setting :kafka, default: {}
+    # Middleware chain that can be expanded with useful middleware steps
+    setting(
+      :middleware,
+      default: false,
+      constructor: ->(middleware) { middleware || WaterDrop::Middleware.new }
+    )
 
     # Configuration method
     # @yield Runs a block of code providing a config singleton instance to it
@@ -63,10 +94,13 @@ module WaterDrop
         yield(config)
 
         merge_kafka_defaults!(config)
-        validate!(config.to_h)
+
+        Contracts::Config.new.validate!(config.to_h, Errors::ConfigurationInvalidError)
 
         ::Rdkafka::Config.logger = config.logger
       end
+
+      self
     end
 
     private
@@ -74,7 +108,7 @@ module WaterDrop
     # Propagates the kafka setting defaults unless they are already present
     # This makes it easier to set some values that users usually don't change but still allows them
     # to overwrite the whole hash if they want to
-    # @param config [Dry::Configurable::Config] dry config of this producer
+    # @param config [Karafka::Core::Configurable::Node] config of this producer
     def merge_kafka_defaults!(config)
       KAFKA_DEFAULTS.each do |key, value|
         next if config.kafka.key?(key)
@@ -82,16 +116,5 @@ module WaterDrop
         config.kafka[key] = value
       end
     end
-
-    # Validates the configuration and if anything is wrong, will raise an exception
-    # @param config_hash [Hash] config hash with setup details
-    # @raise [WaterDrop::Errors::ConfigurationInvalidError] raised when something is wrong with
-    #   the configuration
-    def validate!(config_hash)
-      result = Contracts::Config.new.call(config_hash)
-      return true if result.success?
-
-      raise Errors::ConfigurationInvalidError, result.errors.to_h
-    end
   end
 end

data/lib/waterdrop/contracts/config.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Contracts
+    # Contract with validation rules for WaterDrop configuration details
+    class Config < ::Karafka::Core::Contractable::Contract
+      configure do |config|
+        config.error_messages = YAML.safe_load(
+          File.read(
+            File.join(WaterDrop.gem_root, 'config', 'locales', 'errors.yml')
+          )
+        ).fetch('en').fetch('validations').fetch('config')
+      end
+
+      required(:id) { |val| val.is_a?(String) && !val.empty? }
+      required(:logger) { |val| !val.nil? }
+      required(:deliver) { |val| [true, false].include?(val) }
+      required(:max_payload_size) { |val| val.is_a?(Integer) && val >= 1 }
+      required(:max_wait_timeout) { |val| val.is_a?(Numeric) && val >= 0 }
+      required(:wait_timeout) { |val| val.is_a?(Numeric) && val.positive? }
+      required(:kafka) { |val| val.is_a?(Hash) && !val.empty? }
+      required(:wait_on_queue_full) { |val| [true, false].include?(val) }
+      required(:wait_backoff_on_queue_full) { |val| val.is_a?(Numeric) && val >= 0 }
+      required(:wait_timeout_on_queue_full) { |val| val.is_a?(Numeric) && val >= 0 }
+
+      # rdkafka allows both symbols and strings as keys for config but then casts them to strings
+      # This can be confusing, so we expect all keys to be symbolized
+      virtual do |config, errors|
+        next true unless errors.empty?
+
+        errors = []
+
+        config
+          .fetch(:kafka)
+          .keys
+          .reject { |key| key.is_a?(Symbol) }
+          .each { |key| errors << [[:kafka, key], :kafka_key_must_be_a_symbol] }
+
+        errors
+      end
+    end
+  end
+end

data/lib/waterdrop/contracts/message.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Contracts
+    # Contract with validation rules for validating that all the message options that
+    # we provide to producer are valid and usable
+    class Message < ::Karafka::Core::Contractable::Contract
+      configure do |config|
+        config.error_messages = YAML.safe_load(
+          File.read(
+            File.join(WaterDrop.gem_root, 'config', 'locales', 'errors.yml')
+          )
+        ).fetch('en').fetch('validations').fetch('message')
+      end
+
+      # Regex to check that topic has a valid format
+      TOPIC_REGEXP = /\A(\w|-|\.)+\z/
+
+      private_constant :TOPIC_REGEXP
+
+      attr_reader :max_payload_size
+
+      # @param max_payload_size [Integer] max payload size
+      def initialize(max_payload_size:)
+        super()
+        @max_payload_size = max_payload_size
+      end
+
+      required(:topic) do |val|
+        (val.is_a?(String) || val.is_a?(Symbol)) && TOPIC_REGEXP.match?(val.to_s)
+      end
+
+      required(:payload) { |val| val.nil? || val.is_a?(String) }
+      optional(:key) { |val| val.nil? || (val.is_a?(String) && !val.empty?) }
+      optional(:partition) { |val| val.is_a?(Integer) && val >= -1 }
+      optional(:partition_key) { |val| val.nil? || (val.is_a?(String) && !val.empty?) }
+      optional(:timestamp) { |val| val.nil? || (val.is_a?(Time) || val.is_a?(Integer)) }
+      optional(:headers) { |val| val.nil? || val.is_a?(Hash) }
+
+      virtual do |message, errors|
+        next true unless errors.empty?
+        next true unless message.key?(:headers)
+        next true if message[:headers].nil?
+
+        errors = []
+
+        message.fetch(:headers).each do |key, value|
+          errors << [%i[headers], :invalid_key_type] unless key.is_a?(String)
+          errors << [%i[headers], :invalid_value_type] unless value.is_a?(String)
+        end
+
+        errors
+      end
+
+      virtual do |message, errors, validator|
+        next true unless errors.empty?
+        next if message[:payload].nil? # tombstone payload
+        next true if message[:payload].bytesize <= validator.max_payload_size
+
+        [[%i[payload], :max_size]]
+      end
+    end
+  end
+end

data/lib/waterdrop/contracts/transactional_offset.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Contracts
+    # Contract to ensure that arguments provided to the transactional offset commit are valid
+    # and match our expectations
+    class TransactionalOffset < ::Karafka::Core::Contractable::Contract
+      configure do |config|
+        config.error_messages = YAML.safe_load(
+          File.read(
+            File.join(WaterDrop.gem_root, 'config', 'locales', 'errors.yml')
+          )
+        ).fetch('en').fetch('validations').fetch('transactional_offset')
+      end
+
+      required(:consumer) { |val| val.respond_to?(:consumer_group_metadata_pointer) }
+      required(:message) { |val| val.respond_to?(:topic) && val.respond_to?(:partition) }
+      required(:offset_metadata) { |val| val.is_a?(String) || val.nil? }
+    end
+  end
+end

data/lib/{water_drop → waterdrop}/errors.rb

@@ -25,19 +25,35 @@ module WaterDrop
     # Raised when we want to send a message that is invalid (impossible topic, etc)
     MessageInvalidError = Class.new(BaseError)
 
+    # Raised when we want to commit transactional offset and the input is invalid
+    TransactionalOffsetInvalidError = Class.new(BaseError)
+
     # Raised when we've got an unexpected status. This should never happen. If it does, please
     # contact us as it is an error.
     StatusInvalidError = Class.new(BaseError)
 
-    # Raised when during messages flushing something bad happened
-    class FlushFailureError < BaseError
-      attr_reader :dispatched_messages
+    # Raised when there is an inline error during single message produce operations
+    ProduceError = Class.new(BaseError)
+
+    # Raised when we attempt to perform operation that is only allowed inside of a transaction and
+    # there is no transaction around us
+    TransactionRequiredError = Class.new(BaseError)
+
+    # Raise it within a transaction to abort it
+    # It does not have an `Error` postfix because technically it is not an error as it is used for
+    # graceful transaction aborting
+    AbortTransaction = Class.new(BaseError)
+
+    # Raised when during messages producing something bad happened inline
+    class ProduceManyError < ProduceError
+      attr_reader :dispatched
 
-      # @param dispatched_messages [Array<Rdkafka::Producer::DeliveryHandle>] handlers of the
+      # @param dispatched [Array<Rdkafka::Producer::DeliveryHandle>] handlers of the
       #   messages that we've dispatched
-      def initialize(dispatched_messages)
-        super()
-        @dispatched_messages = dispatched_messages
+      # @param message [String] error message
+      def initialize(dispatched, message)
+        super(message)
+        @dispatched = dispatched
       end
     end
   end

data/lib/waterdrop/helpers/counter.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  # Extra internal helper objects
+  module Helpers
+    # Atomic counter that we can safely increment and decrement without race conditions
+    class Counter
+      # @return [Integer] current value
+      attr_reader :value
+
+      def initialize
+        @value = 0
+        @mutex = Mutex.new
+      end
+
+      # Increments the value by 1
+      def increment
+        @mutex.synchronize { @value += 1 }
+      end
+
+      # Decrements the value by 1
+      def decrement
+        @mutex.synchronize { @value -= 1 }
+      end
+    end
+  end
+end

data/lib/waterdrop/instrumentation/callbacks/delivery.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Instrumentation
+    module Callbacks
+      # Creates a callable that we want to run upon each message delivery or failure
+      #
+      # @note We don't have to provide client_name here as this callback is per client instance
+      #
+      # @note We do not consider `message.purge` as an error for transactional producers, because
+      #   this is a standard behaviour for not yet dispatched messages on aborted transactions.
+      #   We do however still want to instrument it for traceability.
+      class Delivery
+        # Error emitted when a message was not yet dispatched and was purged from the queue
+        RD_KAFKA_RESP_PURGE_QUEUE = -152
+
+        # Error emitted when a message was purged while it was dispatched
+        RD_KAFKA_RESP_PURGE_INFLIGHT = -151
+
+        # Errors related to queue purging that is expected in transactions
+        PURGE_ERRORS = [RD_KAFKA_RESP_PURGE_INFLIGHT, RD_KAFKA_RESP_PURGE_QUEUE].freeze
+
+        private_constant :RD_KAFKA_RESP_PURGE_QUEUE, :RD_KAFKA_RESP_PURGE_INFLIGHT, :PURGE_ERRORS
+
+        # @param producer_id [String] id of the current producer
+        # @param transactional [Boolean] is this handle for a transactional or regular producer
+        # @param monitor [WaterDrop::Instrumentation::Monitor] monitor we are using
+        def initialize(producer_id, transactional, monitor)
+          @producer_id = producer_id
+          @transactional = transactional
+          @monitor = monitor
+        end
+
+        # Emits delivery details to the monitor
+        # @param delivery_report [Rdkafka::Producer::DeliveryReport] delivery report
+        def call(delivery_report)
+          error_code = delivery_report.error.to_i
+
+          if error_code.zero?
+            instrument_acknowledged(delivery_report)
+
+          elsif @transactional && PURGE_ERRORS.include?(error_code)
+            instrument_purged(delivery_report)
+          else
+            instrument_error(delivery_report)
+          end
+        end
+
+        private
+
+        # @param delivery_report [Rdkafka::Producer::DeliveryReport] delivery report
+        def instrument_acknowledged(delivery_report)
+          @monitor.instrument(
+            'message.acknowledged',
+            caller: self,
+            producer_id: @producer_id,
+            offset: delivery_report.offset,
+            partition: delivery_report.partition,
+            topic: delivery_report.topic_name,
+            delivery_report: delivery_report,
+            label: delivery_report.label
+          )
+        end
+
+        # @param delivery_report [Rdkafka::Producer::DeliveryReport] delivery report
+        def instrument_purged(delivery_report)
+          @monitor.instrument(
+            'message.purged',
+            caller: self,
+            error: build_error(delivery_report),
+            producer_id: @producer_id,
+            offset: delivery_report.offset,
+            partition: delivery_report.partition,
+            topic: delivery_report.topic_name,
+            delivery_report: delivery_report,
+            label: delivery_report.label,
+            type: 'librdkafka.dispatch_error'
+          )
+        end
+
+        # @param delivery_report [Rdkafka::Producer::DeliveryReport] delivery report
+        def instrument_error(delivery_report)
+          @monitor.instrument(
+            'error.occurred',
+            caller: self,
+            error: build_error(delivery_report),
+            producer_id: @producer_id,
+            offset: delivery_report.offset,
+            partition: delivery_report.partition,
+            topic: delivery_report.topic_name,
+            delivery_report: delivery_report,
+            label: delivery_report.label,
+            type: 'librdkafka.dispatch_error'
+          )
+        end
+
+        # Builds appropriate rdkafka error
+        # @param delivery_report [Rdkafka::Producer::DeliveryReport] delivery report
+        # @return [::Rdkafka::RdkafkaError]
+        def build_error(delivery_report)
+          ::Rdkafka::RdkafkaError.new(delivery_report.error)
+        end
+      end
+    end
+  end
+end

data/lib/{water_drop → waterdrop}/instrumentation/callbacks/error.rb

@@ -18,15 +18,19 @@ module WaterDrop
         # @param client_name [String] rdkafka client name
         # @param error [Rdkafka::Error] error that occurred
         # @note It will only instrument on errors of the client of our producer
+        # @note When there is a particular message produce error (not internal error), the error
+        #   is shipped via the delivery callback, not via error callback.
         def call(client_name, error)
           # Emit only errors related to our client
           # Same as with statistics (mor explanation there)
           return unless @client_name == client_name
 
           @monitor.instrument(
-            'error.emitted',
+            'error.occurred',
+            caller: self,
+            error: error,
             producer_id: @producer_id,
-            error: error
+            type: 'librdkafka.error'
           )
         end
       end

data/lib/{water_drop → waterdrop}/instrumentation/callbacks/statistics.rb

@@ -17,7 +17,7 @@ module WaterDrop
           @producer_id = producer_id
           @client_name = client_name
           @monitor = monitor
-          @statistics_decorator = StatisticsDecorator.new
+          @statistics_decorator = ::Karafka::Core::Monitoring::StatisticsDecorator.new
         end
 
         # Emits decorated statistics to the monitor

data/lib/{water_drop/instrumentation/stdout_listener.rb → waterdrop/instrumentation/logger_listener.rb}

@@ -1,15 +1,25 @@
 # frozen_string_literal: true
 
 module WaterDrop
+  # WaterDrop instrumentation related module
   module Instrumentation
     # Default listener that hooks up to our instrumentation and uses its events for logging
     # It can be removed/replaced or anything without any harm to the Waterdrop flow
     # @note It is a module as we can use it then as a part of the Karafka framework listener
     #   as well as we can use it standalone
-    class StdoutListener
-      # @param logger [Object] stdout logger we want to use
-      def initialize(logger)
+    class LoggerListener
+      # @param logger [Object] logger we want to use
+      # @param log_messages [Boolean] Should we report the messages content (payload and metadata)
+      #   with each message operation.
+      #
+      #   This can be extensive, especially when producing a lot of messages. We provide this
+      #   despite the fact that we only report payloads in debug, because Rails by default operates
+      #   with debug level. This means, that when working with Rails in development, every single
+      #   payload dispatched will go to logs. In majority of the cases this is extensive and simply
+      #   floods the end user.
+      def initialize(logger, log_messages: true)
         @logger = logger
+        @log_messages = log_messages
       end
 
       # @param event [Dry::Events::Event] event that happened with the details
@@ -17,6 +27,9 @@ module WaterDrop
         message = event[:message]
 
         info(event, "Async producing of a message to '#{message[:topic]}' topic")
+
+        return unless log_messages?
+
         debug(event, message)
       end
 
@@ -25,6 +38,9 @@ module WaterDrop
         message = event[:message]
 
         info(event, "Sync producing of a message to '#{message[:topic]}' topic")
+
+        return unless log_messages?
+
         debug(event, message)
       end
 
@@ -34,6 +50,9 @@ module WaterDrop
         topics_count = messages.map { |message| "'#{message[:topic]}'" }.uniq.count
 
         info(event, "Async producing of #{messages.size} messages to #{topics_count} topics")
+
+        return unless log_messages?
+
         debug(event, messages)
       end
 
@@ -43,6 +62,9 @@ module WaterDrop
         topics_count = messages.map { |message| "'#{message[:topic]}'" }.uniq.count
 
         info(event, "Sync producing of #{messages.size} messages to #{topics_count} topics")
+
+        return unless log_messages?
+
         debug(event, messages)
       end
 
@@ -51,6 +73,9 @@ module WaterDrop
         message = event[:message]
 
         info(event, "Buffering of a message to '#{message[:topic]}' topic")
+
+        return unless log_messages?
+
         debug(event, [message])
       end
 
@@ -59,6 +84,9 @@ module WaterDrop
         messages = event[:messages]
 
         info(event, "Buffering of #{messages.size} messages")
+
+        return unless log_messages?
+
         debug(event, [messages, messages.size])
       end
 
@@ -67,15 +95,9 @@ module WaterDrop
         messages = event[:messages]
 
         info(event, "Async flushing of #{messages.size} messages from the buffer")
-        debug(event, messages)
-      end
 
-      # @param event [Dry::Events::Event] event that happened with the details
-      def on_buffer_flushed_async_error(event)
-        messages = event[:messages]
-        error = event[:error]
+        return unless log_messages?
 
-        error(event, "Async flushing of #{messages.size} failed due to: #{error}")
         debug(event, messages)
       end
 
@@ -84,34 +106,78 @@ module WaterDrop
         messages = event[:messages]
 
         info(event, "Sync flushing of #{messages.size} messages from the buffer")
+
+        return unless log_messages?
+
         debug(event, messages)
       end
 
       # @param event [Dry::Events::Event] event that happened with the details
-      def on_buffer_flushed_sync_error(event)
-        messages = event[:dispatched]
-        error = event[:error]
+      def on_buffer_purged(event)
+        info(event, 'Successfully purging buffer')
+      end
 
-        error(event, "Sync flushing of #{messages.size} failed due to: #{error}")
-        debug(event, messages)
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_producer_closing(event)
+        info(event, 'Closing producer')
       end
 
       # @param event [Dry::Events::Event] event that happened with the details
+      # @note While this says "Closing producer", it produces a nice message with time taken:
+      #   "Closing producer took 12 ms" indicating it happened in the past.
       def on_producer_closed(event)
-        info event, 'Closing producer'
-        debug event, ''
+        info(event, 'Closing producer')
       end
 
       # @param event [Dry::Events::Event] event that happened with the error details
-      def on_error_emitted(event)
+      def on_error_occurred(event)
         error = event[:error]
+        type = event[:type]
+
+        error(event, "Error occurred: #{error} - #{type}")
+      end
 
-        error(event, "Background thread error emitted: #{error}")
-        debug(event, '')
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_transaction_started(event)
+        info(event, 'Starting transaction')
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_transaction_aborted(event)
+        info(event, 'Aborting transaction')
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_transaction_committed(event)
+        info(event, 'Committing transaction')
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_transaction_marked_as_consumed(event)
+        message = event[:message]
+        topic = message.topic
+        partition = message.partition
+        offset = message.offset
+        loc = "#{topic}/#{partition}"
+
+        info(
+          event,
+          "Marking message with offset #{offset} for topic #{loc} as consumed in a transaction"
+        )
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_transaction_finished(event)
+        info(event, 'Processing transaction')
       end
 
       private
 
+      # @return [Boolean] should we report the messages details in the debug mode.
+      def log_messages?
+        @log_messages
+      end
+
       # @param event [Dry::Events::Event] event that happened with the details
       # @param log_message [String] message we want to publish
       def debug(event, log_message)
@@ -121,7 +187,11 @@ module WaterDrop
       # @param event [Dry::Events::Event] event that happened with the details
       # @param log_message [String] message we want to publish
       def info(event, log_message)
-        @logger.info("[#{event[:producer_id]}] #{log_message} took #{event[:time]} ms")
+        if event.payload.key?(:time)
+          @logger.info("[#{event[:producer_id]}] #{log_message} took #{event[:time]} ms")
+        else
+          @logger.info("[#{event[:producer_id]}] #{log_message}")
+        end
       end
 
       # @param event [Dry::Events::Event] event that happened with the details

data/lib/waterdrop/instrumentation/monitor.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Instrumentation
+    # WaterDrop instrumentation monitor that we use to publish events
+    # By default uses our internal notifications bus but can be used with
+    # `ActiveSupport::Notifications` as well
+    class Monitor < ::Karafka::Core::Monitoring::Monitor
+      # @param notifications_bus [Object] either our internal notifications bus or
+      #   `ActiveSupport::Notifications`
+      # @param namespace [String, nil] namespace for events or nil if no namespace
+      def initialize(
+        notifications_bus = WaterDrop::Instrumentation::Notifications.new,
+        namespace = nil
+      )
+        super(notifications_bus, namespace)
+      end
+    end
+  end
+end