waterdrop 2.3.2 → 2.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 154b5f8a103fcfaa47c5d410583d1aa2c0b689b52cd284614b06e7f7a9460179
-  data.tar.gz: c01ae313dbcf2830267438d3dd19f269a0a7bf29cad824af95f4313f6f04368e
+  metadata.gz: b1482f0d2955c43c2ab479372c9cbb4a8c039723121c26b2ff75d0e26700d71e
+  data.tar.gz: 1bb3e675c0c632d4940b2b661bdb75a74c44358a7e53fa9dc7051fdad62218b0
 SHA512:
-  metadata.gz: 33e5ab153d45e265dd6cdb2b3ab9bb63bda7c8c4399097f1c5c130d4c493de635ee7fad957c83b97d6e5fdd6c4c049a0419ca5e0e063f8163478182cde6b92ed
-  data.tar.gz: 3858e9ac8d2f860f888a5445ec8c59d98d35dce199816058da850bfaaf80308ccd40cff2eb5e872eb3ba111259c4c4d07d0a0219e0a641da0ad0c456d184030a
+  metadata.gz: 8921844cc44625916974c63478562b66a7f30a833b6f95e71e8c3448109d673bd3520a8a48bdf0fe7296e5e00b988c15c3dd92cab2e11ee37914f667966a14de
+  data.tar.gz: 6c6038c4f9a718d107fd885e66bf29cc3dde09c6f6e0be2ba41651f3494672ddd925e71574f53ec8ec70d4e766024018f73a43a5fadf99402117ac3a0751b0c0

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # WaterDrop changelog
 
+## 2.3.3 (2022-07-18)
+- Replace `dry-validation` with home-brew validation layer and drop direct dependency on `dry-validation`.
+- Remove indirect dependency on dry-configurable from DataDog listener (no changes required).
+
 ## 2.3.2 (2022-07-17)
 - Replace `dry-configurable` with home-brew config and drop direct dependency on `dry-configurable`.
 

data/Gemfile.lock

@@ -1,10 +1,9 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.3.2)
+    waterdrop (2.3.3)
       concurrent-ruby (>= 1.1)
       dry-monitor (~> 0.5)
-      dry-validation (~> 1.7)
       rdkafka (>= 0.10)
       zeitwerk (~> 2.3)
 
@@ -23,42 +22,16 @@ GEM
     dry-configurable (0.15.0)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.6)
-    dry-container (0.10.0)
-      concurrent-ruby (~> 1.0)
     dry-core (0.8.0)
       concurrent-ruby (~> 1.0)
     dry-events (0.3.0)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.5, >= 0.5)
-    dry-inflector (0.3.0)
-    dry-initializer (3.1.1)
-    dry-logic (1.2.0)
-      concurrent-ruby (~> 1.0)
-      dry-core (~> 0.5, >= 0.5)
     dry-monitor (0.6.1)
       dry-configurable (~> 0.13, >= 0.13.0)
       dry-core (~> 0.5, >= 0.5)
       dry-events (~> 0.2)
       zeitwerk (~> 2.5)
-    dry-schema (1.9.3)
-      concurrent-ruby (~> 1.0)
-      dry-configurable (~> 0.13, >= 0.13.0)
-      dry-core (~> 0.5, >= 0.5)
-      dry-initializer (~> 3.0)
-      dry-logic (~> 1.0)
-      dry-types (~> 1.5)
-    dry-types (1.5.1)
-      concurrent-ruby (~> 1.0)
-      dry-container (~> 0.3)
-      dry-core (~> 0.5, >= 0.5)
-      dry-inflector (~> 0.1, >= 0.1.2)
-      dry-logic (~> 1.0, >= 1.0.2)
-    dry-validation (1.8.1)
-      concurrent-ruby (~> 1.0)
-      dry-container (~> 0.7, >= 0.7.1)
-      dry-core (~> 0.5, >= 0.5)
-      dry-initializer (~> 3.0)
-      dry-schema (~> 1.8, >= 1.8.0)
     factory_bot (6.2.1)
       activesupport (>= 5.0.0)
     ffi (1.15.5)

data/config/errors.yml

@@ -1,7 +1,30 @@
 en:
-  dry_validation:
-    errors:
-      invalid_key_type: all keys need to be of type String
-      invalid_value_type: all values need to be of type String
-      max_payload_size: is more than `max_payload_size` config value
+  validations:
+    config:
+      missing: must be present
+      logger_format: must be present
+      deliver_format: must be boolean
+      id_format: must be a non-empty string
+      max_payload_size_format: must be an integer that is equal or bigger than 1
+      wait_timeout_format: must be a numeric that is bigger than 0
+      max_wait_timeout_format: must be an integer that is equal or bigger than 0
+      kafka_format: must be a hash with symbol based keys
       kafka_key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
+
+    message:
+      missing: must be present
+      partition_format: must be an integer greater or equal to -1
+      topic_format: 'does not match the topic allowed format'
+      partition_key_format: must be a non-empty string
+      timestamp_format: must be either time or integer
+      payload_format: must be string
+      headers_format: must be a hash
+      key_format: must be a non-empty string
+      payload_max_size: is more than `max_payload_size` config value
+      headers_invalid_key_type: all headers keys need to be of type String
+      headers_invalid_value_type: all headers values need to be of type String
+
+    test:
+      missing: must be present
+      nested.id_format: 'is invalid'
+      nested.id2_format: 'is invalid'

data/lib/waterdrop/contractable/contract.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Contractable
+    # Base contract for all the contracts that check data format
+    #
+    # @note This contract does NOT support rules inheritance as it was never needed in Karafka
+    class Contract
+      extend Configurable
+
+      # Yaml based error messages data
+      setting(:error_messages)
+
+      # Class level API definitions
+      class << self
+        # @return [Array<Rule>] all the validation rules defined for a given contract
+        attr_reader :rules
+
+        # Allows for definition of a scope/namespace for nested validations
+        #
+        # @param path [Symbol] path in the hash for nesting
+        # @param block [Proc] nested rule code or more nestings inside
+        #
+        # @example
+        #   nested(:key) do
+        #     required(:inside) { |inside| inside.is_a?(String) }
+        #   end
+        def nested(path, &block)
+          init_accu
+          @nested << path
+          instance_eval(&block)
+          @nested.pop
+        end
+
+        # Defines a rule for a required field (required means, that will automatically create an
+        # error if missing)
+        #
+        # @param keys [Array<Symbol>] single or full path
+        # @param block [Proc] validation rule
+        def required(*keys, &block)
+          init_accu
+          @rules << Rule.new(@nested + keys, :required, block).freeze
+        end
+
+        # @param keys [Array<Symbol>] single or full path
+        # @param block [Proc] validation rule
+        def optional(*keys, &block)
+          init_accu
+          @rules << Rule.new(@nested + keys, :optional, block).freeze
+        end
+
+        # @param block [Proc] validation rule
+        #
+        # @note Virtual rules have different result expectations. Please see contracts or specs for
+        #   details.
+        def virtual(&block)
+          init_accu
+          @rules << Rule.new([], :virtual, block).freeze
+        end
+
+        private
+
+        # Initializes nestings and rules building accumulator
+        def init_accu
+          @nested ||= []
+          @rules ||= []
+        end
+      end
+
+      # Runs the validation
+      #
+      # @param data [Hash] hash with data we want to validate
+      # @return [Result] validaton result
+      def call(data)
+        errors = []
+
+        self.class.rules.map do |rule|
+          case rule.type
+          when :required
+            validate_required(data, rule, errors)
+          when :optional
+            validate_optional(data, rule, errors)
+          when :virtual
+            validate_virtual(data, rule, errors)
+          end
+        end
+
+        Result.new(errors, self)
+      end
+
+      # @param data [Hash] data for validation
+      # @param error_class [Class] error class that should be used when validation fails
+      # @return [Boolean] true
+      # @raise [StandardError] any error provided in the error_class that inherits from the
+      #   standard error
+      def validate!(data, error_class)
+        result = call(data)
+
+        return true if result.success?
+
+        raise error_class, result.errors
+      end
+
+      private
+
+      # Runs validation for rules on fields that are required and adds errors (if any) to the
+      # errors array
+      #
+      # @param data [Hash] input hash
+      # @param rule [Rule] validation rule
+      # @param errors [Array] array with errors from previous rules (if any)
+      def validate_required(data, rule, errors)
+        for_checking = dig(data, rule.path)
+
+        if for_checking.first == :match
+          result = rule.validator.call(for_checking.last, data, errors, self)
+
+          return if result == true
+
+          errors << [rule.path, result || :format]
+        else
+          errors << [rule.path, :missing]
+        end
+      end
+
+      # Runs validation for rules on fields that are optional and adds errors (if any) to the
+      # errors array
+      #
+      # @param data [Hash] input hash
+      # @param rule [Rule] validation rule
+      # @param errors [Array] array with errors from previous rules (if any)
+      def validate_optional(data, rule, errors)
+        for_checking = dig(data, rule.path)
+
+        return unless for_checking.first == :match
+
+        result = rule.validator.call(for_checking.last, data, errors, self)
+
+        return if result == true
+
+        errors << [rule.path, result || :format]
+      end
+
+      # Runs validation for rules on virtual fields (aggregates, etc) and adds errors (if any) to
+      # the errors array
+      #
+      # @param data [Hash] input hash
+      # @param rule [Rule] validation rule
+      # @param errors [Array] array with errors from previous rules (if any)
+      def validate_virtual(data, rule, errors)
+        result = rule.validator.call(data, errors, self)
+
+        return if result == true
+
+        errors.push(*result)
+      end
+
+      # Tries to dig for a given key in a hash and returns it with indication whether or not it was
+      # possible to find it (dig returns nil and we don't know if it wasn't the digged key value)
+      #
+      # @param data [Hash]
+      # @param keys [Array<Symbol>]
+      # @return [Array<Symbol, Object>] array where the first element is `:match` or `:miss` and
+      #   the digged value or nil if not found
+      def dig(data, keys)
+        current = data
+        result = :match
+
+        keys.each do |nesting|
+          unless current.key?(nesting)
+            result = :miss
+
+            break
+          end
+
+          current = current[nesting]
+        end
+
+        [result, current]
+      end
+    end
+  end
+end

data/lib/waterdrop/contractable/result.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Contractable
+    # Representation of a validaton result with resolved error messages
+    class Result
+      attr_reader :errors
+
+      # Builds a result object and remaps (if needed) error keys to proper error messages
+      #
+      # @param errors [Array<Array>] array with sub-arrays with paths and error keys
+      # @param contract [Object] contract that generated the error
+      def initialize(errors, contract)
+        # Short track to skip object allocation for the happy path
+        if errors.empty?
+          @errors = errors
+          return
+        end
+
+        hashed = {}
+
+        errors.each do |error|
+          scope = error.first.map(&:to_s).join('.').to_sym
+
+          # This will allow for usage of custom messages instead of yaml keys if needed
+          hashed[scope] = if error.last.is_a?(String)
+                            error.last
+                          else
+                            build_message(contract, scope, error.last)
+                          end
+        end
+
+        @errors = hashed
+      end
+
+      # @return [Boolean] true if no errors
+      def success?
+        errors.empty?
+      end
+
+      private
+
+      # Builds message based on the error messages
+      # @param contract [Object] contract for which we build the result
+      # @param scope [Symbol] path to the key that has an error
+      # @param error_key [Symbol] error key for yaml errors lookup
+      # @return [String] error message
+      def build_message(contract, scope, error_key)
+        messages = contract.class.config.error_messages
+
+        messages.fetch(error_key.to_s) do
+          messages.fetch("#{scope}_#{error_key}")
+        end
+      end
+    end
+  end
+end

data/lib/waterdrop/contractable/rule.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Contractable
+    # Representation of a single validation rule
+    Rule = Struct.new(:path, :type, :validator)
+  end
+end

data/lib/waterdrop/contractable.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  # Contract layer for WaterDrop and Karafka
+  # It aims to be "dry-validation" like but smaller and easier to handle + without dependencies
+  #
+  # It allows for nested validations, etc
+  #
+  # @note It is thread-safe to run but validations definitions should happen before threads are
+  #   used.
+  module Contractable
+  end
+end

data/lib/waterdrop/contracts/config.rb

@@ -3,27 +3,37 @@
 module WaterDrop
   module Contracts
     # Contract with validation rules for WaterDrop configuration details
-    class Config < Base
-      params do
-        required(:id).filled(:str?)
-        required(:logger).filled
-        required(:deliver).filled(:bool?)
-        required(:max_payload_size).filled(:int?, gteq?: 1)
-        required(:max_wait_timeout).filled(:number?, gteq?: 0)
-        required(:wait_timeout).filled(:number?, gt?: 0)
-        required(:kafka).filled(:hash?)
+    class Config < Contractable::Contract
+      configure do |config|
+        config.error_messages = YAML.safe_load(
+          File.read(
+            File.join(WaterDrop.gem_root, 'config', 'errors.yml')
+          )
+        ).fetch('en').fetch('validations').fetch('config')
       end
 
+      required(:id) { |id| id.is_a?(String) && !id.empty? }
+      required(:logger) { |logger| !logger.nil? }
+      required(:deliver) { |deliver| [true, false].include?(deliver) }
+      required(:max_payload_size) { |ps| ps.is_a?(Integer) && ps >= 1 }
+      required(:max_wait_timeout) { |mwt| mwt.is_a?(Numeric) && mwt >= 0 }
+      required(:wait_timeout) { |wt| wt.is_a?(Numeric) && wt.positive? }
+      required(:kafka) { |kafka| kafka.is_a?(Hash) && !kafka.empty? }
+
       # rdkafka allows both symbols and strings as keys for config but then casts them to strings
       # This can be confusing, so we expect all keys to be symbolized
-      rule(:kafka) do
-        next unless value.is_a?(Hash)
+      virtual do |config, errors|
+        next true unless errors.empty?
+
+        errors = []
 
-        value.each_key do |key|
-          next if key.is_a?(Symbol)
+        config
+          .fetch(:kafka)
+          .keys
+          .reject { |key| key.is_a?(Symbol) }
+          .each { |key| errors << [[:kafka, key], :kafka_key_must_be_a_symbol] }
 
-          key(:"kafka.#{key}").failure(:kafka_key_must_be_a_symbol)
-        end
+        errors
       end
     end
   end

data/lib/waterdrop/contracts/message.rb

@@ -4,36 +4,56 @@ module WaterDrop
   module Contracts
     # Contract with validation rules for validating that all the message options that
     # we provide to producer ale valid and usable
-    class Message < Base
+    class Message < Contractable::Contract
+      configure do |config|
+        config.error_messages = YAML.safe_load(
+          File.read(
+            File.join(WaterDrop.gem_root, 'config', 'errors.yml')
+          )
+        ).fetch('en').fetch('validations').fetch('message')
+      end
+
       # Regex to check that topic has a valid format
       TOPIC_REGEXP = /\A(\w|-|\.)+\z/
 
-      # Checks, that the given value is a string
-      STRING_ASSERTION = ->(value) { value.is_a?(String) }.to_proc
+      private_constant :TOPIC_REGEXP
 
-      private_constant :TOPIC_REGEXP, :STRING_ASSERTION
+      attr_reader :max_payload_size
 
-      option :max_payload_size
-
-      params do
-        required(:topic).filled(:str?, format?: TOPIC_REGEXP)
-        required(:payload).filled(:str?)
-        optional(:key).maybe(:str?, :filled?)
-        optional(:partition).filled(:int?, gteq?: -1)
-        optional(:partition_key).maybe(:str?, :filled?)
-        optional(:timestamp).maybe { time? | int? }
-        optional(:headers).maybe(:hash?)
+      # @param max_payload_size [Integer] max payload size
+      def initialize(max_payload_size:)
+        super()
+        @max_payload_size = max_payload_size
       end
 
-      rule(:headers) do
-        next unless value.is_a?(Hash)
+      required(:topic) { |topic| topic.is_a?(String) && TOPIC_REGEXP.match?(topic) }
+      required(:payload) { |payload| payload.is_a?(String) }
+      optional(:key) { |key| key.nil? || (key.is_a?(String) && !key.empty?) }
+      optional(:partition) { |partition| partition.is_a?(Integer) && partition >= -1 }
+      optional(:partition_key) { |p_key| p_key.nil? || (p_key.is_a?(String) && !p_key.empty?) }
+      optional(:timestamp) { |ts| ts.nil? || (ts.is_a?(Time) || ts.is_a?(Integer)) }
+      optional(:headers) { |headers| headers.nil? || headers.is_a?(Hash) }
+
+      virtual do |config, errors|
+        next true unless errors.empty?
+        next true unless config.key?(:headers)
+        next true if config[:headers].nil?
 
-        key.failure(:invalid_key_type) unless value.keys.all?(&STRING_ASSERTION)
-        key.failure(:invalid_value_type) unless value.values.all?(&STRING_ASSERTION)
+        errors = []
+
+        config.fetch(:headers).each do |key, value|
+          errors << [%i[headers], :invalid_key_type] unless key.is_a?(String)
+          errors << [%i[headers], :invalid_value_type] unless value.is_a?(String)
+        end
+
+        errors
       end
 
-      rule(:payload) do
-        key.failure(:max_payload_size) if value.bytesize > max_payload_size
+      virtual do |config, errors, validator|
+        next true unless errors.empty?
+        next true if config[:payload].bytesize <= validator.max_payload_size
+
+        [[%i[payload], :max_size]]
       end
     end
   end

data/lib/waterdrop/instrumentation/vendors/datadog/listener.rb

@@ -11,26 +11,29 @@ module WaterDrop
         #
         # @note You need to setup the `dogstatsd-ruby` client and assign it
         class Listener
-          include Dry::Configurable
+          include WaterDrop::Configurable
+          extend Forwardable
+
+          def_delegators :config, :client, :rd_kafka_metrics, :namespace, :default_tags
 
           # Value object for storing a single rdkafka metric publishing details
           RdKafkaMetric = Struct.new(:type, :scope, :name, :key_location)
 
           # Namespace under which the DD metrics should be published
-          setting :namespace, default: 'waterdrop', reader: true
+          setting :namespace, default: 'waterdrop'
 
           # Datadog client that we should use to publish the metrics
-          setting :client, reader: true
+          setting :client
 
           # Default tags we want to publish (for example hostname)
           # Format as followed (example for hostname): `["host:#{Socket.gethostname}"]`
-          setting :default_tags, default: [], reader: true
+          setting :default_tags, default: []
 
           # All the rdkafka metrics we want to publish
           #
           # By default we publish quite a lot so this can be tuned
           # Note, that the once with `_d` come from WaterDrop, not rdkafka or Kafka
-          setting :rd_kafka_metrics, reader: true, default: [
+          setting :rd_kafka_metrics, default: [
             # Client metrics
             RdKafkaMetric.new(:count, :root, 'calls', 'tx_d'),
             RdKafkaMetric.new(:histogram, :root, 'queue.size', 'msg_cnt_d'),
@@ -47,8 +50,11 @@ module WaterDrop
             RdKafkaMetric.new(:gauge, :brokers, 'network.latency.p99', %w[rtt p99])
           ].freeze
 
+          configure
+
           # @param block [Proc] configuration block
           def initialize(&block)
+            configure
             setup(&block) if block
           end
 

data/lib/waterdrop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '2.3.2'
+  VERSION = '2.3.3'
 end

data/lib/waterdrop.rb

@@ -3,9 +3,11 @@
 # External components
 # delegate should be removed because we don't need it, we just add it because of ruby-kafka
 %w[
+  forwardable
   concurrent/array
+  concurrent/hash
+  yaml
   dry/monitor/notifications
-  dry-validation
   rdkafka
   json
   zeitwerk

data/waterdrop.gemspec

@@ -18,7 +18,6 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'concurrent-ruby', '>= 1.1'
   spec.add_dependency 'dry-monitor', '~> 0.5'
-  spec.add_dependency 'dry-validation', '~> 1.7'
   spec.add_dependency 'rdkafka', '>= 0.10'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 2.3.2
+  version: 2.3.3
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -34,7 +34,7 @@ cert_chain:
   R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
   pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
   -----END CERTIFICATE-----
-date: 2022-07-17 00:00:00.000000000 Z
+date: 2022-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -64,20 +64,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.5'
-- !ruby/object:Gem::Dependency
-  name: dry-validation
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.7'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.7'
 - !ruby/object:Gem::Dependency
   name: rdkafka
   requirement: !ruby/object:Gem::Requirement
@@ -133,8 +119,11 @@ files:
 - lib/waterdrop/configurable.rb
 - lib/waterdrop/configurable/leaf.rb
 - lib/waterdrop/configurable/node.rb
+- lib/waterdrop/contractable.rb
+- lib/waterdrop/contractable/contract.rb
+- lib/waterdrop/contractable/result.rb
+- lib/waterdrop/contractable/rule.rb
 - lib/waterdrop/contracts.rb
-- lib/waterdrop/contracts/base.rb
 - lib/waterdrop/contracts/config.rb
 - lib/waterdrop/contracts/message.rb
 - lib/waterdrop/errors.rb

data/lib/waterdrop/contracts/base.rb

@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-module WaterDrop
-  module Contracts
-    # Base for all the contracts in WaterDrop
-    class Base < Dry::Validation::Contract
-      config.messages.load_paths << File.join(WaterDrop.gem_root, 'config', 'errors.yml')
-
-      # @param data [Hash] data for validation
-      # @param error_class [Class] error class that should be used when validation fails
-      # @return [Boolean] true
-      # @raise [StandardError] any error provided in the error_class that inherits from the
-      #   standard error
-      def validate!(data, error_class)
-        result = call(data)
-
-        return true if result.success?
-
-        raise error_class, result.errors.to_h
-      end
-    end
-  end
-end