waterdrop 1.4.0

data/certs/mensfeld.pem

@@ -0,0 +1,25 @@
+-----BEGIN CERTIFICATE-----
+MIIEODCCAqCgAwIBAgIBATANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhtYWNp
+ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMjAwODExMDkxNTM3WhcNMjEwODExMDkx
+NTM3WjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
+CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDCpXsCgmINb6lHBXXBdyrgsBPSxC4/
+2H+weJ6L9CruTiv2+2/ZkQGtnLcDgrD14rdLIHK7t0o3EKYlDT5GhD/XUVhI15JE
+N7IqnPUgexe1fbZArwQ51afxz2AmPQN2BkB2oeQHXxnSWUGMhvcEZpfbxCCJH26w
+hS0Ccsma8yxA6hSlGVhFVDuCr7c2L1di6cK2CtIDpfDaWqnVNJEwBYHIxrCoWK5g
+sIGekVt/admS9gRhIMaIBg+Mshth5/DEyWO2QjteTodItlxfTctrfmiAl8X8T5JP
+VXeLp5SSOJ5JXE80nShMJp3RFnGw5fqjX/ffjtISYh78/By4xF3a25HdWH9+qO2Z
+tx0wSGc9/4gqNM0APQnjN/4YXrGZ4IeSjtE+OrrX07l0TiyikzSLFOkZCAp8oBJi
+Fhlosz8xQDJf7mhNxOaZziqASzp/hJTU/tuDKl5+ql2icnMv5iV/i6SlmvU29QNg
+LCV71pUv0pWzN+OZbHZKWepGhEQ3cG9MwvkCAwEAAaN3MHUwCQYDVR0TBAIwADAL
+BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFImGed2AXS070ohfRidiCEhXEUN+MB0GA1Ud
+EQQWMBSBEm1hY2llakBtZW5zZmVsZC5wbDAdBgNVHRIEFjAUgRJtYWNpZWpAbWVu
+c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBAKiHpwoENVrMi94V1zD4o8/6G3AU
+gWz4udkPYHTZLUy3dLznc/sNjdkJFWT3E6NKYq7c60EpJ0m0vAEg5+F5pmNOsvD3
+2pXLj9kisEeYhR516HwXAvtngboUcb75skqvBCU++4Pu7BRAPjO1/ihLSBexbwSS
+fF+J5OWNuyHHCQp+kGPLtXJe2yUYyvSWDj3I2//Vk0VhNOIlaCS1+5/P3ZJThOtm
+zJUBI7h3HgovwRpcnmk2mXTmU4Zx/bCzX8EA6VY0khEvnmiq7S6eBF0H9qH8KyQ6
+EkVLpvmUDFcf/uNaBQdazEMB5jYtwoA8gQlANETNGPi51KlkukhKgaIEDMkBDJOx
+65N7DzmkcyY0/GwjIVIxmRhcrCt1YeCUElmfFx0iida1/YRm6sB2AXqScc1+ECRi
+2DND//YJUikn1zwbz1kT70XmHd97B4Eytpln7K+M1u2g1pHVEPW4owD/ammXNpUy
+nt70FcDD4yxJQ+0YNiHd0N8IcVBM1TMIVctMNQ==
+-----END CERTIFICATE-----

data/config/errors.yml

@@ -0,0 +1,19 @@
+en:
+  dry_validation:
+    errors:
+      broker_schema: >
+        has an invalid format.
+        Expected schema, host and port number.
+        Example: kafka://127.0.0.1:9092 or kafka+ssl://127.0.0.1:9092
+      ssl_client_cert_with_ssl_client_cert_key: >
+        Both ssl_client_cert and ssl_client_cert_key need to be provided.
+      ssl_client_cert_key_with_ssl_client_cert: >
+        Both ssl_client_cert_key and ssl_client_cert need to be provided.
+      ssl_client_cert_chain_with_ssl_client_cert: >
+        Both ssl_client_cert_chain and ssl_client_cert need to be provided.
+      ssl_client_cert_chain_with_ssl_client_cert_key: >
+        Both ssl_client_cert_chain and ssl_client_cert_key need to be provided.
+      ssl_client_cert_key_password_with_ssl_client_cert_key: >
+        Both ssl_client_cert_key_password and ssl_client_cert_key need to be provided.
+      sasl_oauth_token_provider_respond_to_token: >
+        sasl_oauth_token_provider needs to respond to a #token method.

data/lib/water_drop.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+# External components
+# delegate should be removed because we don't need it, we just add it because of ruby-kafka
+%w[
+  delegate
+  json
+  delivery_boy
+  singleton
+  dry-configurable
+  dry/monitor/notifications
+  dry-validation
+  zeitwerk
+].each { |lib| require lib }
+
+# WaterDrop library
+module WaterDrop
+  class << self
+    attr_accessor :logger
+
+    # Sets up the whole configuration
+    # @param [Block] block configuration block
+    def setup(&block)
+      Config.setup(&block)
+      DeliveryBoy.logger = self.logger = config.logger
+      ConfigApplier.call(DeliveryBoy.config, Config.config.to_h)
+    end
+
+    # @return [WaterDrop::Config] config instance
+    def config
+      Config.config
+    end
+
+    # @return [::WaterDrop::Monitor] monitor that we want to use
+    def monitor
+      config.monitor
+    end
+
+    # @return [String] root path of this gem
+    def gem_root
+      Pathname.new(File.expand_path('..', __dir__))
+    end
+  end
+end
+
+Zeitwerk::Loader
+  .for_gem
+  .tap { |loader| loader.ignore("#{__dir__}/waterdrop.rb") }
+  .tap(&:setup)
+  .tap(&:eager_load)

data/lib/water_drop/async_producer.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# WaterDrop library
+module WaterDrop
+  # Async producer for messages
+  class AsyncProducer < BaseProducer
+    # Performs message delivery using deliver_async method
+    # @param message [String] message that we want to send to Kafka
+    # @param options [Hash] options (including topic) for producer
+    # @raise [WaterDrop::Errors::InvalidMessageOptions] raised when message options are
+    #   somehow invalid and we cannot perform delivery because of that
+    def self.call(message, options)
+      attempts_count ||= 0
+      attempts_count += 1
+
+      validate!(options)
+      return unless WaterDrop.config.deliver
+
+      d_method = WaterDrop.config.raise_on_buffer_overflow ? :deliver_async! : :deliver_async
+
+      DeliveryBoy.send(d_method, message, **options)
+    rescue Kafka::Error => e
+      graceful_attempt?(attempts_count, message, options, e) ? retry : raise(e)
+    end
+  end
+end

data/lib/water_drop/base_producer.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  # Base messages producer that contains all the logic that is exactly the same for both
+  # sync and async producers
+  class BaseProducer
+    # Contract for checking the correctness of the provided data that someone wants to
+    # dispatch to Kafka
+    SCHEMA = Contracts::MessageOptions.new.freeze
+
+    private_constant :SCHEMA
+
+    class << self
+      private
+
+      # Runs the message options validations and raises an error if anything is invalid
+      # @param options [Hash] hash that we want to validate
+      # @raise [WaterDrop::Errors::InvalidMessageOptions] raised when message options are
+      #   somehow invalid and we cannot perform delivery because of that
+      def validate!(options)
+        validation_result = SCHEMA.call(options)
+        return true if validation_result.success?
+
+        raise Errors::InvalidMessageOptions, validation_result.errors
+      end
+
+      # Upon failed delivery, we may try to resend a message depending on the attempt number
+      #   or re-raise an error if we're unable to do that after given number of retries
+      #   This method checks that and also instruments errors and retries for the delivery
+      # @param attempts_count [Integer] number of attempt (starting from 1) for the delivery
+      # @param message [String] message that we want to send to Kafka
+      # @param options [Hash] options (including topic) for producer
+      # @param error [Kafka::Error] error that occurred
+      # @return [Boolean] true if this is a graceful attempt and we can retry or false it this
+      #   was the final one and we should deal with the fact, that we cannot deliver a given
+      #   message
+      def graceful_attempt?(attempts_count, message, options, error)
+        scope = "#{to_s.split('::').last.sub('Producer', '_producer').downcase}.call"
+        payload = {
+          caller: self,
+          message: message,
+          options: options,
+          error: error,
+          attempts_count: attempts_count
+        }
+
+        if attempts_count > WaterDrop.config.kafka.max_retries
+          WaterDrop.monitor.instrument("#{scope}.error", payload)
+          false
+        else
+          WaterDrop.monitor.instrument("#{scope}.retry", payload)
+          true
+        end
+      end
+    end
+  end
+end

data/lib/water_drop/config.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+# Configuration and descriptions are based on the delivery boy zendesk gem
+# @see https://github.com/zendesk/delivery_boy
+module WaterDrop
+  # Configuration object for setting up all options required by WaterDrop
+  class Config
+    extend Dry::Configurable
+
+    # Config schema definition
+    # @note We use a single instance not to create new one upon each usage
+    SCHEMA = Contracts::Config.new.freeze
+
+    private_constant :SCHEMA
+
+    # WaterDrop options
+    # option client_id [String] identifier of this producer
+    setting :client_id, 'waterdrop'
+    # option [Instance, nil] logger that we want to use or nil to fallback to ruby-kafka logger
+    setting :logger, Logger.new($stdout, level: Logger::WARN)
+    # option [Instance] monitor that we want to use. See instrumentation part of the README for
+    #   more details
+    setting :monitor, WaterDrop::Instrumentation::Monitor.new
+    # option [Boolean] should we send messages. Setting this to false can be really useful when
+    #   testing and or developing because when set to false, won't actually ping Kafka
+    setting :deliver, true
+    # option [Boolean] if you're producing messages faster than the framework or the network can
+    #   send them off, ruby-kafka might reject them. If that happens, WaterDrop will either raise
+    #   or ignore - this setting manages that behavior. This only applies to async producer as
+    #   sync producer will always raise upon problems
+    setting :raise_on_buffer_overflow, true
+
+    # Settings directly related to the Kafka driver
+    setting :kafka do
+      # option [Array<String>] Array that contains Kafka seed broker hosts with ports
+      setting :seed_brokers
+
+      # Network timeouts
+      # option connect_timeout [Integer] Sets the number of seconds to wait while connecting to
+      # a broker for the first time. When ruby-kafka initializes, it needs to connect to at
+      # least one host.
+      setting :connect_timeout, 10
+      # option socket_timeout [Integer] Sets the number of seconds to wait when reading from or
+      # writing to a socket connection to a broker. After this timeout expires the connection
+      # will be killed. Note that some Kafka operations are by definition long-running, such as
+      # waiting for new messages to arrive in a partition, so don't set this value too low
+      setting :socket_timeout, 30
+
+      # Buffering for async producer
+      # @option [Integer] The maximum number of bytes allowed in the buffer before new messages
+      #   are rejected.
+      setting :max_buffer_bytesize, 10_000_000
+      # @option [Integer] The maximum number of messages allowed in the buffer before new messages
+      #   are rejected.
+      setting :max_buffer_size, 1000
+      # @option [Integer] The maximum number of messages allowed in the queue before new messages
+      #   are rejected. The queue is used to ferry messages from the foreground threads of your
+      #   application to the background thread that buffers and delivers messages.
+      setting :max_queue_size, 1000
+
+      # option [Integer] A timeout executed by a broker when the client is sending messages to it.
+      #   It defines the number of seconds the broker should wait for replicas to acknowledge the
+      #   write before responding to the client with an error. As such, it relates to the
+      #   required_acks setting. It should be set lower than socket_timeout.
+      setting :ack_timeout, 5
+      # option [Integer] The number of seconds between background message
+      #   deliveries. Default is 10 seconds. Disable timer-based background deliveries by
+      #   setting this to 0.
+      setting :delivery_interval, 10
+      # option [Integer] The number of buffered messages that will trigger a background message
+      #   delivery. Default is 100 messages. Disable buffer size based background deliveries by
+      #   setting this to 0.
+      setting :delivery_threshold, 100
+      # option [Boolean]
+      setting :idempotent, false
+      # option [Boolean]
+      setting :transactional, false
+      # option [Integer]
+      setting :transactional_timeout, 60
+
+      # option [Integer] The number of retries when attempting to deliver messages.
+      setting :max_retries, 2
+      # option [Integer]
+      setting :required_acks, -1
+      # option [Integer]
+      setting :retry_backoff, 1
+
+      # option [Integer] The minimum number of messages that must be buffered before compression is
+      #   attempted. By default only one message is required. Only relevant if compression_codec
+      #   is set.
+      setting :compression_threshold, 1
+      # option [Symbol] The codec used to compress messages. Must be either snappy or gzip.
+      setting :compression_codec, nil
+
+      # SSL authentication related settings
+      # option ca_cert [String, nil] SSL CA certificate
+      setting :ssl_ca_cert, nil
+      # option ssl_ca_cert_file_path [String, nil] SSL CA certificate file path
+      setting :ssl_ca_cert_file_path, nil
+      # option ssl_ca_certs_from_system [Boolean] Use the CA certs from your system's default
+      #   certificate store
+      setting :ssl_ca_certs_from_system, false
+      # option ssl_verify_hostname [Boolean] Verify the hostname for client certs
+      setting :ssl_verify_hostname, true
+      # option ssl_client_cert [String, nil] SSL client certificate
+      setting :ssl_client_cert, nil
+      # option ssl_client_cert_key [String, nil] SSL client certificate password
+      setting :ssl_client_cert_key, nil
+      # option sasl_gssapi_principal [String, nil] sasl principal
+      setting :sasl_gssapi_principal, nil
+      # option sasl_gssapi_keytab [String, nil] sasl keytab
+      setting :sasl_gssapi_keytab, nil
+      # option sasl_plain_authzid [String] The authorization identity to use
+      setting :sasl_plain_authzid, ''
+      # option sasl_plain_username [String, nil] The username used to authenticate
+      setting :sasl_plain_username, nil
+      # option sasl_plain_password [String, nil] The password used to authenticate
+      setting :sasl_plain_password, nil
+      # option sasl_scram_username [String, nil] The username used to authenticate
+      setting :sasl_scram_username, nil
+      # option sasl_scram_password [String, nil] The password used to authenticate
+      setting :sasl_scram_password, nil
+      # option sasl_scram_mechanism [String, nil] Scram mechanism, either 'sha256' or 'sha512'
+      setting :sasl_scram_mechanism, nil
+      # option sasl_over_ssl [Boolean] whether to enforce SSL with SASL
+      setting :sasl_over_ssl, true
+      # option ssl_client_cert_chain [String, nil] client cert chain or nil if not used
+      setting :ssl_client_cert_chain, nil
+      # option ssl_client_cert_key_password [String, nil] the password required to read
+      #   the ssl_client_cert_key
+      setting :ssl_client_cert_key_password, nil
+      # @param sasl_oauth_token_provider [Object, nil] OAuthBearer Token Provider instance that
+      #   implements method token.
+      setting :sasl_oauth_token_provider, nil
+    end
+
+    class << self
+      # Configuration method
+      # @yield Runs a block of code providing a config singleton instance to it
+      # @yieldparam [WaterDrop::Config] WaterDrop config instance
+      def setup
+        configure do |config|
+          yield(config)
+          validate!(config.to_h)
+        end
+      end
+
+      private
+
+      # Validates the configuration and if anything is wrong, will raise an exception
+      # @param config_hash [Hash] config hash with setup details
+      # @raise [WaterDrop::Errors::InvalidConfiguration] raised when something is wrong with
+      #   the configuration
+      def validate!(config_hash)
+        validation_result = SCHEMA.call(config_hash)
+        return true if validation_result.success?
+
+        raise Errors::InvalidConfiguration, validation_result.errors.to_h
+      end
+    end
+  end
+end

data/lib/water_drop/config_applier.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  # Engine used to propagate config application to DeliveryBoy with corner case handling
+  module ConfigApplier
+    class << self
+      # @param delivery_boy_config [DeliveryBoy::Config] delivery boy config instance
+      # @param settings [Hash] hash with WaterDrop settings
+      def call(delivery_boy_config, settings)
+        # Recursive lambda for mapping config down to delivery boy
+        settings.each do |key, value|
+          call(delivery_boy_config, value) && next if value.is_a?(Hash)
+
+          # If this is a special case that needs manual setup instead of a direct reassignment
+          if respond_to?(key, true)
+            send(key, delivery_boy_config, value)
+          else
+            # If this setting is our internal one, we should not sync it with the delivery boy
+            next unless delivery_boy_config.respond_to?(:"#{key}=")
+
+            delivery_boy_config.public_send(:"#{key}=", value)
+          end
+        end
+      end
+
+      private
+
+      # Extra setup for the compression codec as it behaves differently than other settings
+      # that are ported 1:1 from ruby-kafka
+      # For some crazy reason, delivery boy requires compression codec as a string, when
+      # ruby-kafka as a symbol. We follow ruby-kafka internal design, so we had to mimic
+      # that by assigning a string version that down the road will be symbolized again
+      # by delivery boy
+      # @param delivery_boy_config [DeliveryBoy::Config] delivery boy config instance
+      # @param codec_name [Symbol] codec name as a symbol
+      def compression_codec(delivery_boy_config, codec_name)
+        # If there is no compression codec, we don't apply anything
+        return unless codec_name
+
+        delivery_boy_config.compression_codec = codec_name.to_s
+      end
+
+      # We use the "seed_brokers" name and DeliveryBoy uses "brokers" so we pass the values
+      #   manually
+      # @param delivery_boy_config [DeliveryBoy::Config] delivery boy config instance
+      # @param seed_brokers [Array<String>] kafka seed brokers
+      def seed_brokers(delivery_boy_config, seed_brokers)
+        delivery_boy_config.brokers = seed_brokers
+      end
+    end
+  end
+end

data/lib/water_drop/contracts.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  # Namespace for all the contracts for config validations
+  module Contracts
+    # Regex to check that topic has a valid format
+    TOPIC_REGEXP = /\A(\w|\-|\.)+\z/.freeze
+  end
+end

data/lib/water_drop/contracts/config.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Contracts
+    # Contract with validation rules for WaterDrop configuration details
+    class Config < Dry::Validation::Contract
+      # Valid uri schemas of Kafka broker url
+      URI_SCHEMES = %w[kafka kafka+ssl plaintext ssl].freeze
+
+      # Available sasl scram mechanism of authentication (plus nil)
+      SASL_SCRAM_MECHANISMS = %w[sha256 sha512].freeze
+
+      # Supported compression codecs
+      COMPRESSION_CODECS = %i[snappy gzip lz4 zstd].freeze
+
+      config.messages.load_paths << File.join(WaterDrop.gem_root, 'config', 'errors.yml')
+
+      class << self
+        private
+
+        # Builder for kafka scoped data custom rules
+        # @param keys [Symbol, Hash] the keys names
+        # @param block [Proc] block we want to run with validations within the kafka scope
+        def kafka_scope_rule(*keys, &block)
+          rule(*[:kafka].product(keys)) do
+            instance_exec(values[:kafka], &block)
+          end
+        end
+      end
+
+      private
+
+      # Uri validator to check if uri is in a Kafka acceptable format
+      # @param uri [String] uri we want to validate
+      # @return [Boolean] true if it is a valid uri, otherwise false
+      def broker_schema?(uri)
+        uri = URI.parse(uri)
+        URI_SCHEMES.include?(uri.scheme) && uri.port
+      rescue URI::InvalidURIError
+        false
+      end
+
+      params do
+        required(:client_id).filled(:str?, format?: Contracts::TOPIC_REGEXP)
+        required(:logger).filled
+        required(:deliver).filled(:bool?)
+        required(:raise_on_buffer_overflow).filled(:bool?)
+
+        required(:kafka).schema do
+          required(:seed_brokers).value(:array, :filled?).each(:str?)
+          required(:connect_timeout).filled(:int?, gt?: 0)
+          required(:socket_timeout).filled(:int?, gt?: 0)
+          required(:compression_threshold).filled(:int?, gteq?: 1)
+          optional(:compression_codec).maybe(included_in?: COMPRESSION_CODECS)
+
+          required(:max_buffer_bytesize).filled(:int?, gt?: 0)
+          required(:max_buffer_size).filled(:int?, gt?: 0)
+          required(:max_queue_size).filled(:int?, gt?: 0)
+
+          required(:ack_timeout).filled(:int?, gt?: 0)
+          required(:delivery_interval).filled(:int?, gteq?: 0)
+          required(:delivery_threshold).filled(:int?, gteq?: 0)
+
+          required(:max_retries).filled(:int?, gteq?: 0)
+          required(:retry_backoff).filled(:int?, gteq?: 0)
+          required(:required_acks).filled(included_in?: [1, 0, -1, :all])
+
+          %i[
+            ssl_ca_cert
+            ssl_ca_cert_file_path
+            ssl_client_cert
+            ssl_client_cert_key
+            ssl_client_cert_chain
+            ssl_client_cert_key_password
+            sasl_gssapi_principal
+            sasl_gssapi_keytab
+            sasl_plain_authzid
+            sasl_plain_username
+            sasl_plain_password
+            sasl_scram_username
+            sasl_scram_password
+          ].each do |encryption_attribute|
+            optional(encryption_attribute).maybe(:str?)
+          end
+
+          optional(:ssl_verify_hostname).maybe(:bool?)
+          optional(:ssl_ca_certs_from_system).maybe(:bool?)
+          optional(:sasl_over_ssl).maybe(:bool?)
+          optional(:sasl_oauth_token_provider).value(:any)
+
+          # It's not with other encryptions as it has some more rules
+          optional(:sasl_scram_mechanism)
+            .maybe(:str?, included_in?: SASL_SCRAM_MECHANISMS)
+        end
+      end
+
+      kafka_scope_rule(:seed_brokers) do |kafka|
+        unless kafka[:seed_brokers].all?(&method(:broker_schema?))
+          key(%i[kafka seed_brokers]).failure(:broker_schema)
+        end
+      end
+
+      kafka_scope_rule(:ssl_client_cert, :ssl_client_cert_key) do |kafka|
+        if kafka[:ssl_client_cert] &&
+           kafka[:ssl_client_cert_key].nil?
+          key(%i[kafka ssl_client_cert_key]).failure(:ssl_client_cert_with_ssl_client_cert_key)
+        end
+      end
+
+      kafka_scope_rule(:ssl_client_cert_key, :ssl_client_cert) do |kafka|
+        if kafka[:ssl_client_cert_key] &&
+           kafka[:ssl_client_cert].nil?
+          key.failure(:ssl_client_cert_key_with_ssl_client_cert)
+        end
+      end
+
+      kafka_scope_rule(:ssl_client_cert_chain, :ssl_client_cert) do |kafka|
+        if kafka[:ssl_client_cert_chain] &&
+           kafka[:ssl_client_cert].nil?
+          key.failure(:ssl_client_cert_chain_with_ssl_client_cert)
+        end
+      end
+
+      kafka_scope_rule(:ssl_client_cert_key_password, :ssl_client_cert_key) do |kafka|
+        if kafka[:ssl_client_cert_key_password] &&
+           kafka[:ssl_client_cert_key].nil?
+          key.failure(:ssl_client_cert_key_password_with_ssl_client_cert_key)
+        end
+      end
+
+      kafka_scope_rule(:sasl_oauth_token_provider) do |kafka|
+        if kafka[:sasl_oauth_token_provider] &&
+           !kafka[:sasl_oauth_token_provider].respond_to?(:token)
+          key.failure(:sasl_oauth_token_provider_respond_to_token)
+        end
+      end
+    end
+  end
+end