waterdrop 1.4.0 → 2.0.0.rc1

data/config/errors.yml

@@ -1,19 +1,6 @@
 en:
   dry_validation:
     errors:
-      broker_schema: >
-        has an invalid format.
-        Expected schema, host and port number.
-        Example: kafka://127.0.0.1:9092 or kafka+ssl://127.0.0.1:9092
-      ssl_client_cert_with_ssl_client_cert_key: >
-        Both ssl_client_cert and ssl_client_cert_key need to be provided.
-      ssl_client_cert_key_with_ssl_client_cert: >
-        Both ssl_client_cert_key and ssl_client_cert need to be provided.
-      ssl_client_cert_chain_with_ssl_client_cert: >
-        Both ssl_client_cert_chain and ssl_client_cert need to be provided.
-      ssl_client_cert_chain_with_ssl_client_cert_key: >
-        Both ssl_client_cert_chain and ssl_client_cert_key need to be provided.
-      ssl_client_cert_key_password_with_ssl_client_cert_key: >
-        Both ssl_client_cert_key_password and ssl_client_cert_key need to be provided.
-      sasl_oauth_token_provider_respond_to_token: >
-        sasl_oauth_token_provider needs to respond to a #token method.
+      invalid_key_type: all keys need to be of type String
+      invalid_value_type: all values need to be of type String
+      max_payload_size: is more than `max_payload_size` config value

data/docker-compose.yml

@@ -0,0 +1,17 @@
+version: '2'
+services:
+  zookeeper:
+    image: wurstmeister/zookeeper
+    ports:
+      - "2181:2181"
+  kafka:
+    image: wurstmeister/kafka:1.0.1
+    ports:
+      - "9092:9092"
+    environment:
+      KAFKA_ADVERTISED_HOST_NAME: localhost
+      KAFKA_ADVERTISED_PORT: 9092
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock

data/lib/water_drop.rb

@@ -3,39 +3,19 @@
 # External components
 # delegate should be removed because we don't need it, we just add it because of ruby-kafka
 %w[
-  delegate
-  json
-  delivery_boy
-  singleton
+  concurrent/array
   dry-configurable
   dry/monitor/notifications
   dry-validation
+  rdkafka
+  json
   zeitwerk
+  securerandom
 ].each { |lib| require lib }
 
 # WaterDrop library
 module WaterDrop
   class << self
-    attr_accessor :logger
-
-    # Sets up the whole configuration
-    # @param [Block] block configuration block
-    def setup(&block)
-      Config.setup(&block)
-      DeliveryBoy.logger = self.logger = config.logger
-      ConfigApplier.call(DeliveryBoy.config, Config.config.to_h)
-    end
-
-    # @return [WaterDrop::Config] config instance
-    def config
-      Config.config
-    end
-
-    # @return [::WaterDrop::Monitor] monitor that we want to use
-    def monitor
-      config.monitor
-    end
-
     # @return [String] root path of this gem
     def gem_root
       Pathname.new(File.expand_path('..', __dir__))

data/lib/water_drop/config.rb

@@ -5,158 +5,57 @@
 module WaterDrop
   # Configuration object for setting up all options required by WaterDrop
   class Config
-    extend Dry::Configurable
-
-    # Config schema definition
-    # @note We use a single instance not to create new one upon each usage
-    SCHEMA = Contracts::Config.new.freeze
-
-    private_constant :SCHEMA
+    include Dry::Configurable
 
     # WaterDrop options
-    # option client_id [String] identifier of this producer
-    setting :client_id, 'waterdrop'
-    # option [Instance, nil] logger that we want to use or nil to fallback to ruby-kafka logger
-    setting :logger, Logger.new($stdout, level: Logger::WARN)
+    #
+    # option [String] id of the producer. This can be helpful when building producer specific
+    #   instrumentation or loggers. It is not the kafka producer id
+    setting(:id, false) { |id| id || SecureRandom.uuid }
+    # option [Instance] logger that we want to use
+    # @note Due to how rdkafka works, this setting is global for all the producers
+    setting(:logger, false) { |logger| logger || Logger.new($stdout, level: Logger::WARN) }
     # option [Instance] monitor that we want to use. See instrumentation part of the README for
     #   more details
-    setting :monitor, WaterDrop::Instrumentation::Monitor.new
+    setting(:monitor, false) { |monitor| monitor || WaterDrop::Instrumentation::Monitor.new }
+    # option [Integer] max payload size allowed for delivery to Kafka
+    setting :max_payload_size, 1_000_012
+    # option [Integer] Wait that long for the delivery report or raise an error if this takes
+    #   longer than the timeout.
+    setting :max_wait_timeout, 5
+    # option [Numeric] how long should we wait between re-checks on the availability of the
+    #   delivery report. In a really robust systems, this describes the min-delivery time
+    #   for a single sync message when produced in isolation
+    setting :wait_timeout, 0.005 # 5 milliseconds
     # option [Boolean] should we send messages. Setting this to false can be really useful when
-    #   testing and or developing because when set to false, won't actually ping Kafka
+    #   testing and or developing because when set to false, won't actually ping Kafka but will
+    #   run all the validations, etc
     setting :deliver, true
-    # option [Boolean] if you're producing messages faster than the framework or the network can
-    #   send them off, ruby-kafka might reject them. If that happens, WaterDrop will either raise
-    #   or ignore - this setting manages that behavior. This only applies to async producer as
-    #   sync producer will always raise upon problems
-    setting :raise_on_buffer_overflow, true
-
-    # Settings directly related to the Kafka driver
-    setting :kafka do
-      # option [Array<String>] Array that contains Kafka seed broker hosts with ports
-      setting :seed_brokers
-
-      # Network timeouts
-      # option connect_timeout [Integer] Sets the number of seconds to wait while connecting to
-      # a broker for the first time. When ruby-kafka initializes, it needs to connect to at
-      # least one host.
-      setting :connect_timeout, 10
-      # option socket_timeout [Integer] Sets the number of seconds to wait when reading from or
-      # writing to a socket connection to a broker. After this timeout expires the connection
-      # will be killed. Note that some Kafka operations are by definition long-running, such as
-      # waiting for new messages to arrive in a partition, so don't set this value too low
-      setting :socket_timeout, 30
-
-      # Buffering for async producer
-      # @option [Integer] The maximum number of bytes allowed in the buffer before new messages
-      #   are rejected.
-      setting :max_buffer_bytesize, 10_000_000
-      # @option [Integer] The maximum number of messages allowed in the buffer before new messages
-      #   are rejected.
-      setting :max_buffer_size, 1000
-      # @option [Integer] The maximum number of messages allowed in the queue before new messages
-      #   are rejected. The queue is used to ferry messages from the foreground threads of your
-      #   application to the background thread that buffers and delivers messages.
-      setting :max_queue_size, 1000
-
-      # option [Integer] A timeout executed by a broker when the client is sending messages to it.
-      #   It defines the number of seconds the broker should wait for replicas to acknowledge the
-      #   write before responding to the client with an error. As such, it relates to the
-      #   required_acks setting. It should be set lower than socket_timeout.
-      setting :ack_timeout, 5
-      # option [Integer] The number of seconds between background message
-      #   deliveries. Default is 10 seconds. Disable timer-based background deliveries by
-      #   setting this to 0.
-      setting :delivery_interval, 10
-      # option [Integer] The number of buffered messages that will trigger a background message
-      #   delivery. Default is 100 messages. Disable buffer size based background deliveries by
-      #   setting this to 0.
-      setting :delivery_threshold, 100
-      # option [Boolean]
-      setting :idempotent, false
-      # option [Boolean]
-      setting :transactional, false
-      # option [Integer]
-      setting :transactional_timeout, 60
-
-      # option [Integer] The number of retries when attempting to deliver messages.
-      setting :max_retries, 2
-      # option [Integer]
-      setting :required_acks, -1
-      # option [Integer]
-      setting :retry_backoff, 1
-
-      # option [Integer] The minimum number of messages that must be buffered before compression is
-      #   attempted. By default only one message is required. Only relevant if compression_codec
-      #   is set.
-      setting :compression_threshold, 1
-      # option [Symbol] The codec used to compress messages. Must be either snappy or gzip.
-      setting :compression_codec, nil
-
-      # SSL authentication related settings
-      # option ca_cert [String, nil] SSL CA certificate
-      setting :ssl_ca_cert, nil
-      # option ssl_ca_cert_file_path [String, nil] SSL CA certificate file path
-      setting :ssl_ca_cert_file_path, nil
-      # option ssl_ca_certs_from_system [Boolean] Use the CA certs from your system's default
-      #   certificate store
-      setting :ssl_ca_certs_from_system, false
-      # option ssl_verify_hostname [Boolean] Verify the hostname for client certs
-      setting :ssl_verify_hostname, true
-      # option ssl_client_cert [String, nil] SSL client certificate
-      setting :ssl_client_cert, nil
-      # option ssl_client_cert_key [String, nil] SSL client certificate password
-      setting :ssl_client_cert_key, nil
-      # option sasl_gssapi_principal [String, nil] sasl principal
-      setting :sasl_gssapi_principal, nil
-      # option sasl_gssapi_keytab [String, nil] sasl keytab
-      setting :sasl_gssapi_keytab, nil
-      # option sasl_plain_authzid [String] The authorization identity to use
-      setting :sasl_plain_authzid, ''
-      # option sasl_plain_username [String, nil] The username used to authenticate
-      setting :sasl_plain_username, nil
-      # option sasl_plain_password [String, nil] The password used to authenticate
-      setting :sasl_plain_password, nil
-      # option sasl_scram_username [String, nil] The username used to authenticate
-      setting :sasl_scram_username, nil
-      # option sasl_scram_password [String, nil] The password used to authenticate
-      setting :sasl_scram_password, nil
-      # option sasl_scram_mechanism [String, nil] Scram mechanism, either 'sha256' or 'sha512'
-      setting :sasl_scram_mechanism, nil
-      # option sasl_over_ssl [Boolean] whether to enforce SSL with SASL
-      setting :sasl_over_ssl, true
-      # option ssl_client_cert_chain [String, nil] client cert chain or nil if not used
-      setting :ssl_client_cert_chain, nil
-      # option ssl_client_cert_key_password [String, nil] the password required to read
-      #   the ssl_client_cert_key
-      setting :ssl_client_cert_key_password, nil
-      # @param sasl_oauth_token_provider [Object, nil] OAuthBearer Token Provider instance that
-      #   implements method token.
-      setting :sasl_oauth_token_provider, nil
-    end
-
-    class << self
-      # Configuration method
-      # @yield Runs a block of code providing a config singleton instance to it
-      # @yieldparam [WaterDrop::Config] WaterDrop config instance
-      def setup
-        configure do |config|
-          yield(config)
-          validate!(config.to_h)
-        end
+    # rdkafka options
+    # @see https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
+    setting :kafka, {}
+
+    # Configuration method
+    # @yield Runs a block of code providing a config singleton instance to it
+    # @yieldparam [WaterDrop::Config] WaterDrop config instance
+    def setup
+      configure do |config|
+        yield(config)
+        validate!(config.to_h)
       end
+    end
 
-      private
+    private
 
-      # Validates the configuration and if anything is wrong, will raise an exception
-      # @param config_hash [Hash] config hash with setup details
-      # @raise [WaterDrop::Errors::InvalidConfiguration] raised when something is wrong with
-      #   the configuration
-      def validate!(config_hash)
-        validation_result = SCHEMA.call(config_hash)
-        return true if validation_result.success?
+    # Validates the configuration and if anything is wrong, will raise an exception
+    # @param config_hash [Hash] config hash with setup details
+    # @raise [WaterDrop::Errors::ConfigurationInvalidError] raised when something is wrong with
+    #   the configuration
+    def validate!(config_hash)
+      result = Contracts::Config.new.call(config_hash)
+      return true if result.success?
 
-        raise Errors::InvalidConfiguration, validation_result.errors.to_h
-      end
+      raise Errors::ConfigurationInvalidError, result.errors.to_h
     end
   end
 end

data/lib/water_drop/contracts.rb

@@ -3,7 +3,5 @@
 module WaterDrop
   # Namespace for all the contracts for config validations
   module Contracts
-    # Regex to check that topic has a valid format
-    TOPIC_REGEXP = /\A(\w|\-|\.)+\z/.freeze
   end
 end

data/lib/water_drop/contracts/config.rb

@@ -4,134 +4,21 @@ module WaterDrop
   module Contracts
     # Contract with validation rules for WaterDrop configuration details
     class Config < Dry::Validation::Contract
-      # Valid uri schemas of Kafka broker url
-      URI_SCHEMES = %w[kafka kafka+ssl plaintext ssl].freeze
+      # Ensure valid format of each seed broker so that rdkafka doesn't fail silently
+      SEED_BROKER_FORMAT_REGEXP = %r{\A([^\:\/,]+:[0-9]+)(,[^\:\/,]+:[0-9]+)*\z}.freeze
 
-      # Available sasl scram mechanism of authentication (plus nil)
-      SASL_SCRAM_MECHANISMS = %w[sha256 sha512].freeze
-
-      # Supported compression codecs
-      COMPRESSION_CODECS = %i[snappy gzip lz4 zstd].freeze
-
-      config.messages.load_paths << File.join(WaterDrop.gem_root, 'config', 'errors.yml')
-
-      class << self
-        private
-
-        # Builder for kafka scoped data custom rules
-        # @param keys [Symbol, Hash] the keys names
-        # @param block [Proc] block we want to run with validations within the kafka scope
-        def kafka_scope_rule(*keys, &block)
-          rule(*[:kafka].product(keys)) do
-            instance_exec(values[:kafka], &block)
-          end
-        end
-      end
-
-      private
-
-      # Uri validator to check if uri is in a Kafka acceptable format
-      # @param uri [String] uri we want to validate
-      # @return [Boolean] true if it is a valid uri, otherwise false
-      def broker_schema?(uri)
-        uri = URI.parse(uri)
-        URI_SCHEMES.include?(uri.scheme) && uri.port
-      rescue URI::InvalidURIError
-        false
-      end
+      private_constant :SEED_BROKER_FORMAT_REGEXP
 
       params do
-        required(:client_id).filled(:str?, format?: Contracts::TOPIC_REGEXP)
+        required(:id).filled(:str?)
         required(:logger).filled
         required(:deliver).filled(:bool?)
-        required(:raise_on_buffer_overflow).filled(:bool?)
+        required(:max_payload_size).filled(:int?, gteq?: 1)
+        required(:max_wait_timeout).filled(:number?, gteq?: 0)
+        required(:wait_timeout).filled(:number?, gt?: 0)
 
         required(:kafka).schema do
-          required(:seed_brokers).value(:array, :filled?).each(:str?)
-          required(:connect_timeout).filled(:int?, gt?: 0)
-          required(:socket_timeout).filled(:int?, gt?: 0)
-          required(:compression_threshold).filled(:int?, gteq?: 1)
-          optional(:compression_codec).maybe(included_in?: COMPRESSION_CODECS)
-
-          required(:max_buffer_bytesize).filled(:int?, gt?: 0)
-          required(:max_buffer_size).filled(:int?, gt?: 0)
-          required(:max_queue_size).filled(:int?, gt?: 0)
-
-          required(:ack_timeout).filled(:int?, gt?: 0)
-          required(:delivery_interval).filled(:int?, gteq?: 0)
-          required(:delivery_threshold).filled(:int?, gteq?: 0)
-
-          required(:max_retries).filled(:int?, gteq?: 0)
-          required(:retry_backoff).filled(:int?, gteq?: 0)
-          required(:required_acks).filled(included_in?: [1, 0, -1, :all])
-
-          %i[
-            ssl_ca_cert
-            ssl_ca_cert_file_path
-            ssl_client_cert
-            ssl_client_cert_key
-            ssl_client_cert_chain
-            ssl_client_cert_key_password
-            sasl_gssapi_principal
-            sasl_gssapi_keytab
-            sasl_plain_authzid
-            sasl_plain_username
-            sasl_plain_password
-            sasl_scram_username
-            sasl_scram_password
-          ].each do |encryption_attribute|
-            optional(encryption_attribute).maybe(:str?)
-          end
-
-          optional(:ssl_verify_hostname).maybe(:bool?)
-          optional(:ssl_ca_certs_from_system).maybe(:bool?)
-          optional(:sasl_over_ssl).maybe(:bool?)
-          optional(:sasl_oauth_token_provider).value(:any)
-
-          # It's not with other encryptions as it has some more rules
-          optional(:sasl_scram_mechanism)
-            .maybe(:str?, included_in?: SASL_SCRAM_MECHANISMS)
-        end
-      end
-
-      kafka_scope_rule(:seed_brokers) do |kafka|
-        unless kafka[:seed_brokers].all?(&method(:broker_schema?))
-          key(%i[kafka seed_brokers]).failure(:broker_schema)
-        end
-      end
-
-      kafka_scope_rule(:ssl_client_cert, :ssl_client_cert_key) do |kafka|
-        if kafka[:ssl_client_cert] &&
-           kafka[:ssl_client_cert_key].nil?
-          key(%i[kafka ssl_client_cert_key]).failure(:ssl_client_cert_with_ssl_client_cert_key)
-        end
-      end
-
-      kafka_scope_rule(:ssl_client_cert_key, :ssl_client_cert) do |kafka|
-        if kafka[:ssl_client_cert_key] &&
-           kafka[:ssl_client_cert].nil?
-          key.failure(:ssl_client_cert_key_with_ssl_client_cert)
-        end
-      end
-
-      kafka_scope_rule(:ssl_client_cert_chain, :ssl_client_cert) do |kafka|
-        if kafka[:ssl_client_cert_chain] &&
-           kafka[:ssl_client_cert].nil?
-          key.failure(:ssl_client_cert_chain_with_ssl_client_cert)
-        end
-      end
-
-      kafka_scope_rule(:ssl_client_cert_key_password, :ssl_client_cert_key) do |kafka|
-        if kafka[:ssl_client_cert_key_password] &&
-           kafka[:ssl_client_cert_key].nil?
-          key.failure(:ssl_client_cert_key_password_with_ssl_client_cert_key)
-        end
-      end
-
-      kafka_scope_rule(:sasl_oauth_token_provider) do |kafka|
-        if kafka[:sasl_oauth_token_provider] &&
-           !kafka[:sasl_oauth_token_provider].respond_to?(:token)
-          key.failure(:sasl_oauth_token_provider_respond_to_token)
+          required(:'bootstrap.servers').filled(:str?, format?: SEED_BROKER_FORMAT_REGEXP)
         end
       end
     end

data/lib/water_drop/contracts/message.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Contracts
+    # Contract with validation rules for validating that all the message options that
+    # we provide to producer ale valid and usable
+    class Message < Dry::Validation::Contract
+      # Regex to check that topic has a valid format
+      TOPIC_REGEXP = /\A(\w|\-|\.)+\z/.freeze
+
+      # Checks, that the given value is a string
+      STRING_ASSERTION = ->(value) { value.is_a?(String) }.to_proc
+
+      private_constant :TOPIC_REGEXP, :STRING_ASSERTION
+
+      config.messages.load_paths << File.join(WaterDrop.gem_root, 'config', 'errors.yml')
+
+      option :max_payload_size
+
+      params do
+        required(:topic).filled(:str?, format?: TOPIC_REGEXP)
+        required(:payload).filled(:str?)
+        optional(:key).maybe(:str?, :filled?)
+        optional(:partition).filled(:int?, gteq?: -1)
+        optional(:timestamp).maybe { time? | int? }
+        optional(:headers).maybe(:hash?)
+      end
+
+      rule(:headers) do
+        next unless value.is_a?(Hash)
+
+        key.failure(:invalid_key_type) unless value.keys.all?(&STRING_ASSERTION)
+        key.failure(:invalid_value_type) unless value.values.all?(&STRING_ASSERTION)
+      end
+
+      rule(:payload) do
+        key.failure(:max_payload_size) if value.bytesize > max_payload_size
+      end
+    end
+  end
+end