waterdrop 1.3.1 → 2.0.0.rc1

data/lib/water_drop/contracts.rb

@@ -3,7 +3,5 @@
 module WaterDrop
   # Namespace for all the contracts for config validations
   module Contracts
-    # Regex to check that topic has a valid format
-    TOPIC_REGEXP = /\A(\w|\-|\.)+\z/.freeze
   end
 end

data/lib/water_drop/contracts/config.rb

@@ -4,134 +4,21 @@ module WaterDrop
   module Contracts
     # Contract with validation rules for WaterDrop configuration details
     class Config < Dry::Validation::Contract
-      # Valid uri schemas of Kafka broker url
-      URI_SCHEMES = %w[kafka kafka+ssl plaintext ssl].freeze
+      # Ensure valid format of each seed broker so that rdkafka doesn't fail silently
+      SEED_BROKER_FORMAT_REGEXP = %r{\A([^\:\/,]+:[0-9]+)(,[^\:\/,]+:[0-9]+)*\z}.freeze
 
-      # Available sasl scram mechanism of authentication (plus nil)
-      SASL_SCRAM_MECHANISMS = %w[sha256 sha512].freeze
-
-      # Supported compression codecs
-      COMPRESSION_CODECS = %i[snappy gzip lz4 zstd].freeze
-
-      config.messages.load_paths << File.join(WaterDrop.gem_root, 'config', 'errors.yml')
-
-      class << self
-        private
-
-        # Builder for kafka scoped data custom rules
-        # @param keys [Symbol, Hash] the keys names
-        # @param block [Proc] block we want to run with validations within the kafka scope
-        def kafka_scope_rule(*keys, &block)
-          rule(*[:kafka].product(keys)) do
-            instance_exec(values[:kafka], &block)
-          end
-        end
-      end
-
-      private
-
-      # Uri validator to check if uri is in a Kafka acceptable format
-      # @param uri [String] uri we want to validate
-      # @return [Boolean] true if it is a valid uri, otherwise false
-      def broker_schema?(uri)
-        uri = URI.parse(uri)
-        URI_SCHEMES.include?(uri.scheme) && uri.port
-      rescue URI::InvalidURIError
-        false
-      end
+      private_constant :SEED_BROKER_FORMAT_REGEXP
 
       params do
-        required(:client_id).filled(:str?, format?: Contracts::TOPIC_REGEXP)
+        required(:id).filled(:str?)
         required(:logger).filled
         required(:deliver).filled(:bool?)
-        required(:raise_on_buffer_overflow).filled(:bool?)
+        required(:max_payload_size).filled(:int?, gteq?: 1)
+        required(:max_wait_timeout).filled(:number?, gteq?: 0)
+        required(:wait_timeout).filled(:number?, gt?: 0)
 
         required(:kafka).schema do
-          required(:seed_brokers).value(:array, :filled?).each(:str?)
-          required(:connect_timeout).filled(:int?, gt?: 0)
-          required(:socket_timeout).filled(:int?, gt?: 0)
-          required(:compression_threshold).filled(:int?, gteq?: 1)
-          optional(:compression_codec).maybe(included_in?: COMPRESSION_CODECS)
-
-          required(:max_buffer_bytesize).filled(:int?, gt?: 0)
-          required(:max_buffer_size).filled(:int?, gt?: 0)
-          required(:max_queue_size).filled(:int?, gt?: 0)
-
-          required(:ack_timeout).filled(:int?, gt?: 0)
-          required(:delivery_interval).filled(:int?, gteq?: 0)
-          required(:delivery_threshold).filled(:int?, gteq?: 0)
-
-          required(:max_retries).filled(:int?, gteq?: 0)
-          required(:retry_backoff).filled(:int?, gteq?: 0)
-          required(:required_acks).filled(included_in?: [1, 0, -1, :all])
-
-          %i[
-            ssl_ca_cert
-            ssl_ca_cert_file_path
-            ssl_client_cert
-            ssl_client_cert_key
-            ssl_client_cert_chain
-            ssl_client_cert_key_password
-            sasl_gssapi_principal
-            sasl_gssapi_keytab
-            sasl_plain_authzid
-            sasl_plain_username
-            sasl_plain_password
-            sasl_scram_username
-            sasl_scram_password
-          ].each do |encryption_attribute|
-            optional(encryption_attribute).maybe(:str?)
-          end
-
-          optional(:ssl_verify_hostname).maybe(:bool?)
-          optional(:ssl_ca_certs_from_system).maybe(:bool?)
-          optional(:sasl_over_ssl).maybe(:bool?)
-          optional(:sasl_oauth_token_provider).value(:any)
-
-          # It's not with other encryptions as it has some more rules
-          optional(:sasl_scram_mechanism)
-            .maybe(:str?, included_in?: SASL_SCRAM_MECHANISMS)
-        end
-      end
-
-      kafka_scope_rule(:seed_brokers) do |kafka|
-        unless kafka[:seed_brokers].all?(&method(:broker_schema?))
-          key(%i[kafka seed_brokers]).failure(:broker_schema)
-        end
-      end
-
-      kafka_scope_rule(:ssl_client_cert, :ssl_client_cert_key) do |kafka|
-        if kafka[:ssl_client_cert] &&
-           kafka[:ssl_client_cert_key].nil?
-          key(%i[kafka ssl_client_cert_key]).failure(:ssl_client_cert_with_ssl_client_cert_key)
-        end
-      end
-
-      kafka_scope_rule(:ssl_client_cert_key, :ssl_client_cert) do |kafka|
-        if kafka[:ssl_client_cert_key] &&
-           kafka[:ssl_client_cert].nil?
-          key.failure(:ssl_client_cert_key_with_ssl_client_cert)
-        end
-      end
-
-      kafka_scope_rule(:ssl_client_cert_chain, :ssl_client_cert) do |kafka|
-        if kafka[:ssl_client_cert_chain] &&
-           kafka[:ssl_client_cert].nil?
-          key.failure(:ssl_client_cert_chain_with_ssl_client_cert)
-        end
-      end
-
-      kafka_scope_rule(:ssl_client_cert_key_password, :ssl_client_cert_key) do |kafka|
-        if kafka[:ssl_client_cert_key_password] &&
-           kafka[:ssl_client_cert_key].nil?
-          key.failure(:ssl_client_cert_key_password_with_ssl_client_cert_key)
-        end
-      end
-
-      kafka_scope_rule(:sasl_oauth_token_provider) do |kafka|
-        if kafka[:sasl_oauth_token_provider] &&
-           !kafka[:sasl_oauth_token_provider].respond_to?(:token)
-          key.failure(:sasl_oauth_token_provider_respond_to_token)
+          required(:'bootstrap.servers').filled(:str?, format?: SEED_BROKER_FORMAT_REGEXP)
         end
       end
     end

data/lib/water_drop/contracts/message.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Contracts
+    # Contract with validation rules for validating that all the message options that
+    # we provide to producer ale valid and usable
+    class Message < Dry::Validation::Contract
+      # Regex to check that topic has a valid format
+      TOPIC_REGEXP = /\A(\w|\-|\.)+\z/.freeze
+
+      # Checks, that the given value is a string
+      STRING_ASSERTION = ->(value) { value.is_a?(String) }.to_proc
+
+      private_constant :TOPIC_REGEXP, :STRING_ASSERTION
+
+      config.messages.load_paths << File.join(WaterDrop.gem_root, 'config', 'errors.yml')
+
+      option :max_payload_size
+
+      params do
+        required(:topic).filled(:str?, format?: TOPIC_REGEXP)
+        required(:payload).filled(:str?)
+        optional(:key).maybe(:str?, :filled?)
+        optional(:partition).filled(:int?, gteq?: -1)
+        optional(:timestamp).maybe { time? | int? }
+        optional(:headers).maybe(:hash?)
+      end
+
+      rule(:headers) do
+        next unless value.is_a?(Hash)
+
+        key.failure(:invalid_key_type) unless value.keys.all?(&STRING_ASSERTION)
+        key.failure(:invalid_value_type) unless value.values.all?(&STRING_ASSERTION)
+      end
+
+      rule(:payload) do
+        key.failure(:max_payload_size) if value.bytesize > max_payload_size
+      end
+    end
+  end
+end

data/lib/water_drop/errors.rb

@@ -7,12 +7,37 @@ module WaterDrop
     BaseError = Class.new(StandardError)
 
     # Raised when configuration doesn't match with validation contract
-    InvalidConfiguration = Class.new(BaseError)
+    ConfigurationInvalidError = Class.new(BaseError)
 
-    # Raised when we try to send message with invalid options
-    InvalidMessageOptions = Class.new(BaseError)
+    # Raised when we want to use a producer that was not configured
+    ProducerNotConfiguredError = Class.new(BaseError)
 
-    # Raised when want to hook up to an event that is not registered and supported
-    UnregisteredMonitorEvent = Class.new(BaseError)
+    # Raised when we want to reconfigure a producer that was already configured
+    ProducerAlreadyConfiguredError = Class.new(BaseError)
+
+    # Raised when trying to use connected producer from a forked child process
+    # Producers cannot be used in forks if they were already used in the child processes
+    ProducerUsedInParentProcess = Class.new(BaseError)
+
+    # Raised when there was an attempt to use a closed producer
+    ProducerClosedError = Class.new(BaseError)
+
+    # Raised when we want to send a message that is invalid (impossible topic, etc)
+    MessageInvalidError = Class.new(BaseError)
+
+    # Raised when we've got an unexpected status. This should never happen. If it does, please
+    # contact us as it is an error.
+    StatusInvalidError = Class.new(BaseError)
+
+    # Raised when during messages flushing something bad happened
+    class FlushFailureError < BaseError
+      attr_reader :dispatched_messages
+
+      # @param dispatched_messages [Array<Rdkafka::Producer::DeliveryHandle>] handlers of the
+      #   messages that we've dispatched
+      def initialize(dispatched_messages)
+        @dispatched_messages = dispatched_messages
+      end
+    end
   end
 end

data/lib/water_drop/instrumentation.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  # Namespace for all the things related with WaterDrop instrumentation process
+  module Instrumentation
+  end
+end

data/lib/water_drop/instrumentation/monitor.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 module WaterDrop
-  # Namespace for all the things related with WaterDrop instrumentation process
   module Instrumentation
     # Monitor is used to hookup external monitoring services to monitor how WaterDrop works
     # Since it is a pub-sub based on dry-monitor, you can use as many subscribers/loggers at the
@@ -12,34 +11,28 @@ module WaterDrop
     class Monitor < Dry::Monitor::Notifications
       # List of events that we support in the system and to which a monitor client can hook up
       # @note The non-error once support timestamp benchmarking
-      BASE_EVENTS = %w[
-        async_producer.call.error
-        async_producer.call.retry
-        sync_producer.call.error
-        sync_producer.call.retry
+      EVENTS = %w[
+        producer.closed
+        message.produced_async
+        message.produced_sync
+        messages.produced_async
+        messages.produced_sync
+        message.buffered
+        messages.buffered
+        message.acknowledged
+        buffer.flushed_async
+        buffer.flushed_async.error
+        buffer.flushed_sync
+        buffer.flushed_sync.error
+        statistics.emitted
       ].freeze
 
-      private_constant :BASE_EVENTS
+      private_constant :EVENTS
 
       # @return [WaterDrop::Instrumentation::Monitor] monitor instance for system instrumentation
       def initialize
         super(:waterdrop)
-        BASE_EVENTS.each(&method(:register_event))
-      end
-
-      # Allows us to subscribe to events with a code that will be yielded upon events
-      # @param event_name_or_listener [String, Object] name of the event we want to subscribe to
-      #   or a listener if we decide to go with object listener
-      def subscribe(event_name_or_listener)
-        return super unless event_name_or_listener.is_a?(String)
-        return super if available_events.include?(event_name_or_listener)
-
-        raise Errors::UnregisteredMonitorEvent, event_name_or_listener
-      end
-
-      # @return [Array<String>] names of available events to which we can subscribe
-      def available_events
-        __bus__.events.keys
+        EVENTS.each(&method(:register_event))
       end
     end
   end

data/lib/water_drop/instrumentation/stdout_listener.rb

@@ -7,38 +7,119 @@ module WaterDrop
     # @note It is a module as we can use it then as a part of the Karafka framework listener
     #   as well as we can use it standalone
     class StdoutListener
-      # Log levels that we use in this particular listener
-      USED_LOG_LEVELS = %i[
-        info
-        error
-      ].freeze
-
-      %i[
-        sync_producer
-        async_producer
-      ].each do |producer_type|
-        error_name = :"on_#{producer_type}_call_error"
-        retry_name = :"on_#{producer_type}_call_retry"
-
-        define_method error_name do |event|
-          options = event[:options]
-          error = event[:error]
-          error "Delivery failure to: #{options} because of #{error}"
-        end
-
-        define_method retry_name do |event|
-          attempts_count = event[:attempts_count]
-          options = event[:options]
-          error = event[:error]
-
-          info "Attempt #{attempts_count} of delivery to: #{options} because of #{error}"
-        end
-      end
-
-      USED_LOG_LEVELS.each do |log_level|
-        define_method log_level do |*args|
-          WaterDrop.logger.send(log_level, *args)
-        end
+      # @param logger [Object] stdout logger we want to use
+      def initialize(logger)
+        @logger = logger
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_message_produced_async(event)
+        message = event[:message]
+
+        info(event, "Async producing of a message to '#{message[:topic]}' topic")
+        debug(event, message)
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_message_produced_sync(event)
+        message = event[:message]
+
+        info(event, "Sync producing of a message to '#{message[:topic]}' topic")
+        debug(event, message)
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_messages_produced_async(event)
+        messages = event[:messages]
+        topics_count = messages.map { |message| "'#{message[:topic]}'" }.uniq.count
+
+        info(event, "Async producing of #{messages.size} messages to #{topics_count} topics")
+        debug(event, messages)
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_messages_produced_sync(event)
+        messages = event[:messages]
+        topics_count = messages.map { |message| "'#{message[:topic]}'" }.uniq.count
+
+        info(event, "Sync producing of #{messages.size} messages to #{topics_count} topics")
+        debug(event, messages)
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_message_buffered(event)
+        message = event[:message]
+
+        info(event, "Buffering of a message to '#{message[:topic]}' topic")
+        debug(event, [message, event[:producer].messages.size])
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_messages_buffered(event)
+        messages = event[:messages]
+
+        info(event, "Buffering of #{messages.size} messages")
+        debug(event, [messages, event[:producer].messages.size])
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_buffer_flushed_async(event)
+        messages = event[:messages]
+
+        info(event, "Async flushing of #{messages.size} messages from the buffer")
+        debug(event, messages)
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_buffer_flushed_async_error(event)
+        messages = event[:messages]
+        error = event[:error]
+
+        error(event, "Async flushing of #{messages.size} failed due to: #{error}")
+        debug(event, messages)
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_buffer_flushed_sync(event)
+        messages = event[:messages]
+
+        info(event, "Sync flushing of #{messages.size} messages from the buffer")
+        debug(event, messages)
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_buffer_flushed_sync_error(event)
+        messages = event[:dispatched]
+        error = event[:error]
+
+        error(event, "Sync flushing of #{messages.size} failed due to: #{error}")
+        debug(event, messages)
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_producer_closed(event)
+        info event, 'Closing producer'
+        debug event, event[:producer].messages.size
+      end
+
+      private
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      # @param log_message [String] message we want to publish
+      def debug(event, log_message)
+        @logger.debug("[#{event[:producer].id}] #{log_message}")
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      # @param log_message [String] message we want to publish
+      def info(event, log_message)
+        @logger.info("[#{event[:producer].id}] #{log_message} took #{event[:time]} ms")
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      # @param log_message [String] message we want to publish
+      def error(event, log_message)
+        @logger.error("[#{event[:producer].id}] #{log_message}")
       end
     end
   end