waterdrop 1.4.4 → 2.0.0.rc1

data/lib/water_drop/contracts/config.rb

@@ -4,134 +4,21 @@ module WaterDrop
   module Contracts
     # Contract with validation rules for WaterDrop configuration details
     class Config < Dry::Validation::Contract
-      # Valid uri schemas of Kafka broker url
-      URI_SCHEMES = %w[kafka kafka+ssl plaintext ssl].freeze
+      # Ensure valid format of each seed broker so that rdkafka doesn't fail silently
+      SEED_BROKER_FORMAT_REGEXP = %r{\A([^\:\/,]+:[0-9]+)(,[^\:\/,]+:[0-9]+)*\z}.freeze
 
-      # Available sasl scram mechanism of authentication (plus nil)
-      SASL_SCRAM_MECHANISMS = %w[sha256 sha512].freeze
-
-      # Supported compression codecs
-      COMPRESSION_CODECS = %i[snappy gzip lz4 zstd].freeze
-
-      config.messages.load_paths << File.join(WaterDrop.gem_root, 'config', 'errors.yml')
-
-      class << self
-        private
-
-        # Builder for kafka scoped data custom rules
-        # @param keys [Symbol, Hash] the keys names
-        # @param block [Proc] block we want to run with validations within the kafka scope
-        def kafka_scope_rule(*keys, &block)
-          rule(*[:kafka].product(keys)) do
-            instance_exec(values[:kafka], &block)
-          end
-        end
-      end
-
-      private
-
-      # Uri validator to check if uri is in a Kafka acceptable format
-      # @param uri [String] uri we want to validate
-      # @return [Boolean] true if it is a valid uri, otherwise false
-      def broker_schema?(uri)
-        uri = URI.parse(uri)
-        URI_SCHEMES.include?(uri.scheme) && uri.port
-      rescue URI::InvalidURIError
-        false
-      end
+      private_constant :SEED_BROKER_FORMAT_REGEXP
 
       params do
-        required(:client_id).filled(:str?, format?: Contracts::TOPIC_REGEXP)
+        required(:id).filled(:str?)
         required(:logger).filled
         required(:deliver).filled(:bool?)
-        required(:raise_on_buffer_overflow).filled(:bool?)
+        required(:max_payload_size).filled(:int?, gteq?: 1)
+        required(:max_wait_timeout).filled(:number?, gteq?: 0)
+        required(:wait_timeout).filled(:number?, gt?: 0)
 
         required(:kafka).schema do
-          required(:seed_brokers).value(:array, :filled?).each(:str?)
-          required(:connect_timeout).filled(:int?, gt?: 0)
-          required(:socket_timeout).filled(:int?, gt?: 0)
-          required(:compression_threshold).filled(:int?, gteq?: 1)
-          optional(:compression_codec).maybe(included_in?: COMPRESSION_CODECS)
-
-          required(:max_buffer_bytesize).filled(:int?, gt?: 0)
-          required(:max_buffer_size).filled(:int?, gt?: 0)
-          required(:max_queue_size).filled(:int?, gt?: 0)
-
-          required(:ack_timeout).filled(:int?, gt?: 0)
-          required(:delivery_interval).filled(:int?, gteq?: 0)
-          required(:delivery_threshold).filled(:int?, gteq?: 0)
-
-          required(:max_retries).filled(:int?, gteq?: 0)
-          required(:retry_backoff).filled(:int?, gteq?: 0)
-          required(:required_acks).filled(included_in?: [1, 0, -1, :all])
-
-          %i[
-            ssl_ca_cert
-            ssl_ca_cert_file_path
-            ssl_client_cert
-            ssl_client_cert_key
-            ssl_client_cert_chain
-            ssl_client_cert_key_password
-            sasl_gssapi_principal
-            sasl_gssapi_keytab
-            sasl_plain_authzid
-            sasl_plain_username
-            sasl_plain_password
-            sasl_scram_username
-            sasl_scram_password
-          ].each do |encryption_attribute|
-            optional(encryption_attribute).maybe(:str?)
-          end
-
-          optional(:ssl_verify_hostname).maybe(:bool?)
-          optional(:ssl_ca_certs_from_system).maybe(:bool?)
-          optional(:sasl_over_ssl).maybe(:bool?)
-          optional(:sasl_oauth_token_provider).value(:any)
-
-          # It's not with other encryptions as it has some more rules
-          optional(:sasl_scram_mechanism)
-            .maybe(:str?, included_in?: SASL_SCRAM_MECHANISMS)
-        end
-      end
-
-      kafka_scope_rule(:seed_brokers) do |kafka|
-        unless kafka[:seed_brokers].all?(&method(:broker_schema?))
-          key(%i[kafka seed_brokers]).failure(:broker_schema)
-        end
-      end
-
-      kafka_scope_rule(:ssl_client_cert, :ssl_client_cert_key) do |kafka|
-        if kafka[:ssl_client_cert] &&
-           kafka[:ssl_client_cert_key].nil?
-          key(%i[kafka ssl_client_cert_key]).failure(:ssl_client_cert_with_ssl_client_cert_key)
-        end
-      end
-
-      kafka_scope_rule(:ssl_client_cert_key, :ssl_client_cert) do |kafka|
-        if kafka[:ssl_client_cert_key] &&
-           kafka[:ssl_client_cert].nil?
-          key.failure(:ssl_client_cert_key_with_ssl_client_cert)
-        end
-      end
-
-      kafka_scope_rule(:ssl_client_cert_chain, :ssl_client_cert) do |kafka|
-        if kafka[:ssl_client_cert_chain] &&
-           kafka[:ssl_client_cert].nil?
-          key.failure(:ssl_client_cert_chain_with_ssl_client_cert)
-        end
-      end
-
-      kafka_scope_rule(:ssl_client_cert_key_password, :ssl_client_cert_key) do |kafka|
-        if kafka[:ssl_client_cert_key_password] &&
-           kafka[:ssl_client_cert_key].nil?
-          key.failure(:ssl_client_cert_key_password_with_ssl_client_cert_key)
-        end
-      end
-
-      kafka_scope_rule(:sasl_oauth_token_provider) do |kafka|
-        if kafka[:sasl_oauth_token_provider] &&
-           !kafka[:sasl_oauth_token_provider].respond_to?(:token)
-          key.failure(:sasl_oauth_token_provider_respond_to_token)
+          required(:'bootstrap.servers').filled(:str?, format?: SEED_BROKER_FORMAT_REGEXP)
         end
       end
     end

data/lib/water_drop/contracts/message.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Contracts
+    # Contract with validation rules for validating that all the message options that
+    # we provide to producer ale valid and usable
+    class Message < Dry::Validation::Contract
+      # Regex to check that topic has a valid format
+      TOPIC_REGEXP = /\A(\w|\-|\.)+\z/.freeze
+
+      # Checks, that the given value is a string
+      STRING_ASSERTION = ->(value) { value.is_a?(String) }.to_proc
+
+      private_constant :TOPIC_REGEXP, :STRING_ASSERTION
+
+      config.messages.load_paths << File.join(WaterDrop.gem_root, 'config', 'errors.yml')
+
+      option :max_payload_size
+
+      params do
+        required(:topic).filled(:str?, format?: TOPIC_REGEXP)
+        required(:payload).filled(:str?)
+        optional(:key).maybe(:str?, :filled?)
+        optional(:partition).filled(:int?, gteq?: -1)
+        optional(:timestamp).maybe { time? | int? }
+        optional(:headers).maybe(:hash?)
+      end
+
+      rule(:headers) do
+        next unless value.is_a?(Hash)
+
+        key.failure(:invalid_key_type) unless value.keys.all?(&STRING_ASSERTION)
+        key.failure(:invalid_value_type) unless value.values.all?(&STRING_ASSERTION)
+      end
+
+      rule(:payload) do
+        key.failure(:max_payload_size) if value.bytesize > max_payload_size
+      end
+    end
+  end
+end

data/lib/water_drop/contracts.rb

@@ -3,7 +3,5 @@
 module WaterDrop
   # Namespace for all the contracts for config validations
   module Contracts
-    # Regex to check that topic has a valid format
-    TOPIC_REGEXP = /\A(\w|-|\.)+\z/
   end
 end

data/lib/water_drop/errors.rb

@@ -7,12 +7,37 @@ module WaterDrop
     BaseError = Class.new(StandardError)
 
     # Raised when configuration doesn't match with validation contract
-    InvalidConfiguration = Class.new(BaseError)
+    ConfigurationInvalidError = Class.new(BaseError)
 
-    # Raised when we try to send message with invalid options
-    InvalidMessageOptions = Class.new(BaseError)
+    # Raised when we want to use a producer that was not configured
+    ProducerNotConfiguredError = Class.new(BaseError)
 
-    # Raised when want to hook up to an event that is not registered and supported
-    UnregisteredMonitorEvent = Class.new(BaseError)
+    # Raised when we want to reconfigure a producer that was already configured
+    ProducerAlreadyConfiguredError = Class.new(BaseError)
+
+    # Raised when trying to use connected producer from a forked child process
+    # Producers cannot be used in forks if they were already used in the child processes
+    ProducerUsedInParentProcess = Class.new(BaseError)
+
+    # Raised when there was an attempt to use a closed producer
+    ProducerClosedError = Class.new(BaseError)
+
+    # Raised when we want to send a message that is invalid (impossible topic, etc)
+    MessageInvalidError = Class.new(BaseError)
+
+    # Raised when we've got an unexpected status. This should never happen. If it does, please
+    # contact us as it is an error.
+    StatusInvalidError = Class.new(BaseError)
+
+    # Raised when during messages flushing something bad happened
+    class FlushFailureError < BaseError
+      attr_reader :dispatched_messages
+
+      # @param dispatched_messages [Array<Rdkafka::Producer::DeliveryHandle>] handlers of the
+      #   messages that we've dispatched
+      def initialize(dispatched_messages)
+        @dispatched_messages = dispatched_messages
+      end
+    end
   end
 end

data/lib/water_drop/instrumentation/monitor.rb

@@ -11,34 +11,28 @@ module WaterDrop
     class Monitor < Dry::Monitor::Notifications
       # List of events that we support in the system and to which a monitor client can hook up
       # @note The non-error once support timestamp benchmarking
-      BASE_EVENTS = %w[
-        async_producer.call.error
-        async_producer.call.retry
-        sync_producer.call.error
-        sync_producer.call.retry
+      EVENTS = %w[
+        producer.closed
+        message.produced_async
+        message.produced_sync
+        messages.produced_async
+        messages.produced_sync
+        message.buffered
+        messages.buffered
+        message.acknowledged
+        buffer.flushed_async
+        buffer.flushed_async.error
+        buffer.flushed_sync
+        buffer.flushed_sync.error
+        statistics.emitted
       ].freeze
 
-      private_constant :BASE_EVENTS
+      private_constant :EVENTS
 
       # @return [WaterDrop::Instrumentation::Monitor] monitor instance for system instrumentation
       def initialize
         super(:waterdrop)
-        BASE_EVENTS.each(&method(:register_event))
-      end
-
-      # Allows us to subscribe to events with a code that will be yielded upon events
-      # @param event_name_or_listener [String, Object] name of the event we want to subscribe to
-      #   or a listener if we decide to go with object listener
-      def subscribe(event_name_or_listener)
-        return super unless event_name_or_listener.is_a?(String)
-        return super if available_events.include?(event_name_or_listener)
-
-        raise Errors::UnregisteredMonitorEvent, event_name_or_listener
-      end
-
-      # @return [Array<String>] names of available events to which we can subscribe
-      def available_events
-        __bus__.events.keys
+        EVENTS.each(&method(:register_event))
       end
     end
   end

data/lib/water_drop/instrumentation/stdout_listener.rb

@@ -7,38 +7,119 @@ module WaterDrop
     # @note It is a module as we can use it then as a part of the Karafka framework listener
     #   as well as we can use it standalone
     class StdoutListener
-      # Log levels that we use in this particular listener
-      USED_LOG_LEVELS = %i[
-        info
-        error
-      ].freeze
-
-      %i[
-        sync_producer
-        async_producer
-      ].each do |producer_type|
-        error_name = :"on_#{producer_type}_call_error"
-        retry_name = :"on_#{producer_type}_call_retry"
-
-        define_method error_name do |event|
-          options = event[:options]
-          error = event[:error]
-          error "Delivery failure to: #{options} because of #{error}"
-        end
-
-        define_method retry_name do |event|
-          attempts_count = event[:attempts_count]
-          options = event[:options]
-          error = event[:error]
-
-          info "Attempt #{attempts_count} of delivery to: #{options} because of #{error}"
-        end
-      end
-
-      USED_LOG_LEVELS.each do |log_level|
-        define_method log_level do |*args|
-          WaterDrop.logger.send(log_level, *args)
-        end
+      # @param logger [Object] stdout logger we want to use
+      def initialize(logger)
+        @logger = logger
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_message_produced_async(event)
+        message = event[:message]
+
+        info(event, "Async producing of a message to '#{message[:topic]}' topic")
+        debug(event, message)
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_message_produced_sync(event)
+        message = event[:message]
+
+        info(event, "Sync producing of a message to '#{message[:topic]}' topic")
+        debug(event, message)
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_messages_produced_async(event)
+        messages = event[:messages]
+        topics_count = messages.map { |message| "'#{message[:topic]}'" }.uniq.count
+
+        info(event, "Async producing of #{messages.size} messages to #{topics_count} topics")
+        debug(event, messages)
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_messages_produced_sync(event)
+        messages = event[:messages]
+        topics_count = messages.map { |message| "'#{message[:topic]}'" }.uniq.count
+
+        info(event, "Sync producing of #{messages.size} messages to #{topics_count} topics")
+        debug(event, messages)
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_message_buffered(event)
+        message = event[:message]
+
+        info(event, "Buffering of a message to '#{message[:topic]}' topic")
+        debug(event, [message, event[:producer].messages.size])
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_messages_buffered(event)
+        messages = event[:messages]
+
+        info(event, "Buffering of #{messages.size} messages")
+        debug(event, [messages, event[:producer].messages.size])
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_buffer_flushed_async(event)
+        messages = event[:messages]
+
+        info(event, "Async flushing of #{messages.size} messages from the buffer")
+        debug(event, messages)
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_buffer_flushed_async_error(event)
+        messages = event[:messages]
+        error = event[:error]
+
+        error(event, "Async flushing of #{messages.size} failed due to: #{error}")
+        debug(event, messages)
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_buffer_flushed_sync(event)
+        messages = event[:messages]
+
+        info(event, "Sync flushing of #{messages.size} messages from the buffer")
+        debug(event, messages)
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_buffer_flushed_sync_error(event)
+        messages = event[:dispatched]
+        error = event[:error]
+
+        error(event, "Sync flushing of #{messages.size} failed due to: #{error}")
+        debug(event, messages)
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_producer_closed(event)
+        info event, 'Closing producer'
+        debug event, event[:producer].messages.size
+      end
+
+      private
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      # @param log_message [String] message we want to publish
+      def debug(event, log_message)
+        @logger.debug("[#{event[:producer].id}] #{log_message}")
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      # @param log_message [String] message we want to publish
+      def info(event, log_message)
+        @logger.info("[#{event[:producer].id}] #{log_message} took #{event[:time]} ms")
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      # @param log_message [String] message we want to publish
+      def error(event, log_message)
+        @logger.error("[#{event[:producer].id}] #{log_message}")
       end
     end
   end

data/lib/water_drop/producer/async.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  class Producer
+    # Component for asynchronous producer operations
+    module Async
+      # Produces a message to Kafka and does not wait for results
+      #
+      # @param message [Hash] hash that complies with the {Contracts::Message} contract
+      #
+      # @return [Rdkafka::Producer::DeliveryHandle] delivery handle that might return the report
+      #
+      # @raise [Rdkafka::RdkafkaError] When adding the message to rdkafka's queue failed
+      # @raise [Errors::MessageInvalidError] When provided message details are invalid and the
+      #   message could not be sent to Kafka
+      def produce_async(message)
+        ensure_active!
+        validate_message!(message)
+
+        @monitor.instrument(
+          'message.produced_async',
+          producer: self,
+          message: message
+        ) { client.produce(**message) }
+      end
+
+      # Produces many messages to Kafka and does not wait for them to be delivered
+      #
+      # @param messages [Array<Hash>] array with messages that comply with the
+      #   {Contracts::Message} contract
+      #
+      # @return [Array<Rdkafka::Producer::DeliveryHandle>] deliveries handles
+      #
+      # @raise [Rdkafka::RdkafkaError] When adding the messages to rdkafka's queue failed
+      # @raise [Errors::MessageInvalidError] When any of the provided messages details are invalid
+      #   and the message could not be sent to Kafka
+      def produce_many_async(messages)
+        ensure_active!
+        messages.each { |message| validate_message!(message) }
+
+        @monitor.instrument(
+          'messages.produced_async',
+          producer: self,
+          messages: messages
+        ) do
+          messages.map { |message| client.produce(**message) }
+        end
+      end
+    end
+  end
+end

data/lib/water_drop/producer/buffer.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  class Producer
+    # Component for buffered operations
+    module Buffer
+      # Exceptions we catch when dispatching messages from a buffer
+      RESCUED_ERRORS = [
+        Rdkafka::RdkafkaError,
+        Rdkafka::Producer::DeliveryHandle::WaitTimeoutError
+      ].freeze
+
+      private_constant :RESCUED_ERRORS
+
+      # Adds given message into the internal producer buffer without flushing it to Kafka
+      #
+      # @param message [Hash] hash that complies with the {Contracts::Message} contract
+      # @raise [Errors::MessageInvalidError] When provided message details are invalid and the
+      #   message could not be sent to Kafka
+      def buffer(message)
+        ensure_active!
+        validate_message!(message)
+
+        @monitor.instrument(
+          'message.buffered',
+          producer: self,
+          message: message
+        ) { @messages << message }
+      end
+
+      # Adds given messages into the internal producer buffer without flushing them to Kafka
+      #
+      # @param messages [Array<Hash>] array with messages that comply with the
+      #   {Contracts::Message} contract
+      # @raise [Errors::MessageInvalidError] When any of the provided messages details are invalid
+      #   and the message could not be sent to Kafka
+      def buffer_many(messages)
+        ensure_active!
+        messages.each { |message| validate_message!(message) }
+
+        @monitor.instrument(
+          'messages.buffered',
+          producer: self,
+          messages: messages
+        ) do
+          messages.each { |message| @messages << message }
+          messages
+        end
+      end
+
+      # Flushes the internal buffer to Kafka in an async way
+      # @return [Array<Rdkafka::Producer::DeliveryHandle>] delivery handles for messages that were
+      #   flushed
+      def flush_async
+        ensure_active!
+
+        @monitor.instrument(
+          'buffer.flushed_async',
+          producer: self,
+          messages: @messages
+        ) { flush(false) }
+      end
+
+      # Flushes the internal buffer to Kafka in a sync way
+      # @return [Array<Rdkafka::Producer::DeliveryReport>] delivery reports for messages that were
+      #   flushed
+      def flush_sync
+        ensure_active!
+
+        @monitor.instrument(
+          'buffer.flushed_sync',
+          producer: self,
+          messages: @messages
+        ) { flush(true) }
+      end
+
+      private
+
+      # Method for triggering the buffer
+      # @param sync [Boolean] should it flush in a sync way
+      # @return [Array<Rdkafka::Producer::DeliveryHandle, Rdkafka::Producer::DeliveryReport>]
+      #   delivery handles for async or delivery reports for sync
+      # @raise [Errors::FlushFailureError] when there was a failure in flushing
+      # @note We use this method underneath to provide a different instrumentation for sync and
+      #   async flushing within the public API
+      def flush(sync)
+        data_for_dispatch = nil
+        dispatched = []
+
+        @buffer_mutex.synchronize do
+          data_for_dispatch = @messages
+          @messages = Concurrent::Array.new
+        end
+
+        dispatched = data_for_dispatch.map { |message| client.produce(**message) }
+
+        return dispatched unless sync
+
+        dispatched.map do |handler|
+          handler.wait(
+            max_wait_timeout: @config.max_wait_timeout,
+            wait_timeout: @config.wait_timeout
+          )
+        end
+      rescue *RESCUED_ERRORS => e
+        key = sync ? 'buffer.flushed_sync.error' : 'buffer.flush_async.error'
+        @monitor.instrument(key, producer: self, error: e, dispatched: dispatched)
+
+        raise Errors::FlushFailureError.new(dispatched)
+      end
+    end
+  end
+end

data/lib/water_drop/producer/builder.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  class Producer
+    # Class used to construct the rdkafka producer client
+    class Builder
+      # @param producer [Producer] not yet configured producer for which we want to
+      #   build the client
+      # @param config [Object] dry-configurable based configuration object
+      # @return [Rdkafka::Producer, Producer::DummyClient] raw rdkafka producer or a dummy producer
+      #   when we don't want to dispatch any messages
+      def call(producer, config)
+        return DummyClient.new unless config.deliver
+
+        Rdkafka::Config.logger = config.logger
+        Rdkafka::Config.statistics_callback = build_statistics_callback(producer, config.monitor)
+
+        client = Rdkafka::Config.new(config.kafka.to_h).producer
+        client.delivery_callback = build_delivery_callback(producer, config.monitor)
+        client
+      end
+
+      private
+
+      # Creates a proc that we want to run upon each successful message delivery
+      #
+      # @param producer [Producer]
+      # @param monitor [Object] monitor we want to use
+      # @return [Proc] delivery callback
+      def build_delivery_callback(producer, monitor)
+        lambda do |delivery_report|
+          monitor.instrument(
+            'message.acknowledged',
+            producer: producer,
+            offset: delivery_report.offset,
+            partition: delivery_report.partition
+          )
+        end
+      end
+
+      # Creates a proc that we want to run upon each statistics callback execution
+      #
+      # @param producer [Producer]
+      # @param monitor [Object] monitor we want to use
+      # @return [Proc] statistics callback
+      # @note We decorate the statistics with our own decorator because some of the metrics from
+      #   rdkafka are absolute. For example number of sent messages increases not in reference to
+      #   previous statistics emit but from the beginning of the process. We decorate it with diff
+      #   of all the numeric values against the data from the previous callback emit
+      def build_statistics_callback(producer, monitor)
+        statistics_decorator = StatisticsDecorator.new
+
+        lambda do |statistics|
+          monitor.instrument(
+            'statistics.emitted',
+            producer: producer,
+            statistics: statistics_decorator.call(statistics)
+          )
+        end
+      end
+    end
+  end
+end