waterdrop 1.2.5 → 1.3.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7c29b47026833fe48dd22e56da12560ce33a2cb29a7714f1e0984e17c1c9625f
-  data.tar.gz: 4465d3fa293f99f264405284efe9b423032bb9add85a264c728fbf118d29eb55
+  metadata.gz: 29f20d39798734c2c3530de25b44ff2d8987236adccde1c5fe8b04c624487e4b
+  data.tar.gz: 9bd463eac4f7e02a7aad4681e8bcc4e52f809673ea7c6ddbab8c0e0c4d722d52
 SHA512:
-  metadata.gz: f371d28ae61be1a38689e66c500269d3851d3984445a5a2d6d470431d86c376f98648cef22f5e13509e2fba3f16cfa516cd8a0de380496fb8e1cda9c1a4a629f
-  data.tar.gz: 01c8d90d3279b6cc632f1e09808e4fdde7e8d35292802587e6dce325e2ab5cf5f44697e9171ead920460b58cfa4c552b9869be86a9a3cca6aca984ef4fa49f52
+  metadata.gz: 883ded4f2e72051543258d96d5c9a9ffd32785a7fb10f9e90c17460fd55616faf812f57711efb75a61355c27296ebf7b25a8c50536df39d7c4b3683cf2f8aac1
+  data.tar.gz: 55e13aedda6a2626e6078259a4d3e3da0c99405259bf0c549c66e4bf3e88e6671e7390a8f4e995308b9229138e4c80eace5039d1c4f22f8f0008ff44b55f0485

data/{.coditsu.yml → .coditsu/ci.yml}

@@ -1,3 +1,3 @@
+repository_id: '76b3a28c-d72b-44c7-870b-e5217e2f0d5c'
 api_key: <%= ENV['CODITSU_API_KEY'] %>
 api_secret: <%= ENV['CODITSU_API_SECRET'] %>
-repository_id: <%= ENV['CODITSU_REPOSITORY_ID'] %>

data/.github/FUNDING.yml

@@ -0,0 +1 @@
+open_collective: karafka

data/.gitignore

@@ -7,10 +7,11 @@ app.god
 
 # minimal Rails specific artifacts
 db/*.sqlite3
-/log/*
+/log/*.log
 /tmp/*
 *.gem
 *.~
+/.coditsu/local.yml
 
 # various artifacts
 **.war

data/.ruby-version

@@ -1 +1 @@
-2.6.1
+2.6.3

data/.travis.yml

@@ -2,7 +2,6 @@ services:
   - docker
 
 dist: trusty
-sudo: false
 cache: bundler
 
 git:
@@ -19,30 +18,19 @@ test: &test
 jobs:
   include:
     - <<: *test
-      rvm: 2.6.1
+      rvm: 2.6.3
     - <<: *test
-      rvm: 2.5.3
+      rvm: 2.5.5
     - <<: *test
-      rvm: 2.4.5
-    - <<: *test
-      rvm: 2.3.8
+      rvm: 2.4.6
 
     - stage: coditsu
       language: ruby
-      rvm: 2.6.1
+      rvm: 2.6.3
       before_install:
         - gem update --system
         - gem install bundler
-      before_script:
-        - docker create -v /sources --name sources alpine:3.4 /bin/true
-        - docker cp ./ sources:/sources
-      script: >
-        docker run
-        -e CODITSU_API_KEY
-        -e CODITSU_API_SECRET
-        -e CODITSU_REPOSITORY_ID
-        --volumes-from sources
-        coditsu/build-runner:latest
+      script: \curl -sSL https://api.coditsu.io/run/ci | bash
 
 stages:
   - coditsu

data/CHANGELOG.md

@@ -1,5 +1,24 @@
 # WaterDrop changelog
 
+## 1.3.0.rc1 (2019-07-31)
+- Drop Ruby 2.3 support
+- Drop support for Kafka 0.10 in favor of native support for Kafka 0.11.
+- Ruby 2.6.3 support
+- Support message headers
+- `sasl_over_ssl` support
+- Unlock Ruby Kafka + provide support for 0.7 only
+- #60 - Rename listener to StdoutListener
+- Drop support for Kafka 0.10 in favor of native support for Kafka 0.11.
+- Support ruby-kafka 0.7
+- Support message headers
+- `sasl_over_ssl` support
+- `ssl_client_cert_key_password` support
+- #87 - Make stdout listener as instance
+- Use Zeitwerk for gem code loading
+- #93 - zstd compression support
+- #99 - schemas are renamed to contracts
+- Bump delivery_boy (0.2.7 => 0.2.8)
+
 ## 1.2.5
 - Bump deps to match Karafka
 - drop jruby support

data/Gemfile

@@ -4,6 +4,8 @@ source 'https://rubygems.org'
 
 gemspec
 
+gem 'dry-validation'
+
 group :test do
   gem 'rspec'
   gem 'simplecov'

data/Gemfile.lock

@@ -1,92 +1,102 @@
 PATH
   remote: .
   specs:
-    waterdrop (1.2.5)
+    waterdrop (1.3.0.rc1)
       delivery_boy (~> 0.2)
       dry-configurable (~> 0.8)
       dry-monitor (~> 0.3)
-      dry-validation (~> 0.11)
-      null-logger (~> 0.1)
-      ruby-kafka (>= 0.6)
+      dry-validation (~> 1.2)
+      ruby-kafka (>= 0.7.8)
+      zeitwerk (~> 2.1)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    concurrent-ruby (1.1.4)
-    delivery_boy (0.2.7)
-      king_konf (~> 0.2)
-      ruby-kafka (~> 0.5)
+    concurrent-ruby (1.1.5)
+    delivery_boy (0.2.8)
+      king_konf (~> 0.3)
+      ruby-kafka (~> 0.7.8)
     diff-lcs (1.3)
     digest-crc (0.4.1)
-    docile (1.3.1)
-    dry-configurable (0.8.0)
+    docile (1.3.2)
+    dry-configurable (0.8.3)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.4, >= 0.4.7)
-    dry-container (0.7.0)
+    dry-container (0.7.2)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.1, >= 0.1.3)
-    dry-core (0.4.7)
+    dry-core (0.4.8)
       concurrent-ruby (~> 1.0)
-    dry-equalizer (0.2.1)
-    dry-events (0.1.0)
+    dry-equalizer (0.2.2)
+    dry-events (0.2.0)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.4)
       dry-equalizer (~> 0.2)
     dry-inflector (0.1.2)
-    dry-logic (0.5.0)
-      dry-container (~> 0.2, >= 0.2.6)
+    dry-initializer (3.0.1)
+    dry-logic (1.0.2)
+      concurrent-ruby (~> 1.0)
       dry-core (~> 0.2)
       dry-equalizer (~> 0.2)
-    dry-monitor (0.3.0)
+    dry-monitor (0.3.1)
       dry-configurable (~> 0.5)
       dry-core (~> 0.4)
       dry-equalizer (~> 0.2)
       dry-events (~> 0.1)
-    dry-types (0.14.0)
+    dry-schema (1.3.1)
+      concurrent-ruby (~> 1.0)
+      dry-configurable (~> 0.8, >= 0.8.3)
+      dry-core (~> 0.4)
+      dry-equalizer (~> 0.2)
+      dry-initializer (~> 3.0)
+      dry-logic (~> 1.0)
+      dry-types (~> 1.0)
+    dry-types (1.1.1)
       concurrent-ruby (~> 1.0)
       dry-container (~> 0.3)
       dry-core (~> 0.4, >= 0.4.4)
-      dry-equalizer (~> 0.2)
+      dry-equalizer (~> 0.2, >= 0.2.2)
       dry-inflector (~> 0.1, >= 0.1.2)
-      dry-logic (~> 0.5, >= 0.5)
-    dry-validation (0.13.0)
+      dry-logic (~> 1.0, >= 1.0.2)
+    dry-validation (1.2.1)
       concurrent-ruby (~> 1.0)
-      dry-configurable (~> 0.1, >= 0.1.3)
-      dry-core (~> 0.2, >= 0.2.1)
+      dry-container (~> 0.7, >= 0.7.1)
+      dry-core (~> 0.4)
       dry-equalizer (~> 0.2)
-      dry-logic (~> 0.5, >= 0.5.0)
-      dry-types (~> 0.14, >= 0.14)
-    json (2.1.0)
+      dry-initializer (~> 3.0)
+      dry-schema (~> 1.0, >= 1.3.1)
+    json (2.2.0)
     king_konf (0.3.7)
-    null-logger (0.1.5)
     rspec (3.8.0)
       rspec-core (~> 3.8.0)
       rspec-expectations (~> 3.8.0)
       rspec-mocks (~> 3.8.0)
-    rspec-core (3.8.0)
+    rspec-core (3.8.2)
       rspec-support (~> 3.8.0)
-    rspec-expectations (3.8.2)
+    rspec-expectations (3.8.4)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
-    rspec-mocks (3.8.0)
+    rspec-mocks (3.8.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
-    rspec-support (3.8.0)
-    ruby-kafka (0.7.5)
+    rspec-support (3.8.2)
+    ruby-kafka (0.7.9)
       digest-crc
-    simplecov (0.16.1)
+    simplecov (0.17.0)
       docile (~> 1.1)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
+    zeitwerk (2.1.9)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
+  dry-validation
   rspec
   simplecov
   waterdrop!
 
 BUNDLED WITH
-   2.0.1
+   2.0.2

data/README.md

@@ -11,7 +11,7 @@ It is:
 
  - Thread safe
  - Supports sync and async producers
- - Working with 0.10.1+ Kafka
+ - Working with 0.11+ Kafka
 
 ## Installation
 
@@ -31,15 +31,6 @@ and run
 bundle install
 ```
 
-## Kafka 0.10 or prior
-
-If you're using Kafka 0.10, please lock `ruby-kafka` gem in your Gemfile to version `0.6.8`:
-
-```ruby
-gem 'waterdrop'
-gem 'ruby-kafka', '~> 0.6.8'
-```
-
 ## Setup
 
 WaterDrop is a complex tool, that contains multiple configuration options. To keep everything organized, all the configuration options were divided into two groups:
@@ -113,6 +104,7 @@ Both ```SyncProducer``` and ```AsyncProducer``` accept following options:
 | ```partition```     | false    | Integer    | A specific partition number that should be written to               |
 | ```partition_key``` | false    | String     | A string that can be used to deterministically select the partition |
 | ```create_time```   | false    | Time       | The timestamp that should be set on the message                     |
+| ```headers```       | false    | Hash       | Headers for the message                                             |
 
 Keep in mind, that message you want to send should be either binary or stringified (to_s, to_json, etc).
 

data/certs/mensfeld.pem

@@ -0,0 +1,25 @@
+-----BEGIN CERTIFICATE-----
+MIIEODCCAqCgAwIBAgIBATANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhtYWNp
+ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMTkwNzMwMTQ1NDU0WhcNMjAwNzI5MTQ1
+NDU0WjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
+CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQC9fCwtaHZG2SyyNXiH8r0QbJQx/xxl
+dkvwWz9QGJO+O8rEx20FB1Ab+MVkfOscwIv5jWpmk1U9whzDPl1uFtIbgu+sk+Zb
+uQlZyK/DPN6c+/BbBL+RryTBRyvkPLoCVwm7uxc/JZ1n4AI6eF4cCZ2ieZ9QgQbU
+MQs2QPqs9hT50Ez/40GnOdadVfiDDGz+NME2C4ms0BriXwZ1tcRTfJIHe2xjIbbb
+y5qRGfsLKcgMzvLQR24olixyX1MR0s4+Wveq3QL/gBhL4veUcv+UABJA8IJR0kyB
+seHHutusiwZ1v3SjjjW1xLLrc2ARV0mgCb0WaK2T4iA3oFTGLh6Ydz8LNl31KQFv
+94nRd8IhmJxrhQ6dQ/WT9IXoa5S9lfT5lPJeINemH4/6QPABzf9W2IZlCdI9wCdB
+TBaw57MKneGAYZiKjw6OALSy2ltQUCl3RqFl3VP7n8uFy1U987Q5VIIQ3O1UUsQD
+Oe/h+r7GUU4RSPKgPlrwvW9bD/UQ+zF51v8CAwEAAaN3MHUwCQYDVR0TBAIwADAL
+BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFJNIBHdfEUD7TqHqIer2YhWaWhwcMB0GA1Ud
+EQQWMBSBEm1hY2llakBtZW5zZmVsZC5wbDAdBgNVHRIEFjAUgRJtYWNpZWpAbWVu
+c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBAKA4eqko6BTNhlysip6rfBkVTGri
+ZXsL+kRb2hLvsQJS/kLyM21oMlu+LN0aPj3qEFR8mE/YeDD8rLAfruBRTltPNbR7
+xA5eE1gkxY5LfExUtK3b2wPqfmo7mZgfcsMwfYg/tUXw1WpBCnrhAJodpGH6SXmp
+A40qFUZst0vjiOoO+aTblIHPmMJXoZ3K42dTlNKlEiDKUWMRKSgpjjYGEYalFNWI
+hHfCz2r8L2t+dYdMZg1JGbEkq4ADGsAA8ioZIpJd7V4hI17u5TCdi7X5wh/0gN0E
+CgP+nLox3D+l2q0QuQEkayr+auFYkzTCkF+BmEk1D0Ru4mcf3F4CJvEmW4Pzbjqt
+i1tsCWPtJ4E/UUKnKaWKqGbjrjHJ0MuShYzHkodox5IOiCXIQg+1+YSzfXUV6WEK
+KJG/fhg1JV5vVDdVy6x+tv5SQ5ctU0feCsVfESi3rE3zRd+nvzE9HcZ5aXeL1UtJ
+nT5Xrioegu2w1jPyVEgyZgTZC5rvD0nNS5sFNQ==
+-----END CERTIFICATE-----

data/config/errors.yml

@@ -1,6 +1,19 @@
 en:
-  errors:
-    broker_schema?: >
-      has an invalid format.
-      Expected schema, host and port number.
-      Example: kafka://127.0.0.1:9092 or kafka+ssl://127.0.0.1:9092
+  dry_validation:
+    errors:
+      broker_schema: >
+        has an invalid format.
+        Expected schema, host and port number.
+        Example: kafka://127.0.0.1:9092 or kafka+ssl://127.0.0.1:9092
+      ssl_client_cert_with_ssl_client_cert_key: >
+        Both ssl_client_cert and ssl_client_cert_key need to be provided.
+      ssl_client_cert_key_with_ssl_client_cert: >
+        Both ssl_client_cert_key and ssl_client_cert need to be provided.
+      ssl_client_cert_chain_with_ssl_client_cert: >
+        Both ssl_client_cert_chain and ssl_client_cert need to be provided.
+      ssl_client_cert_chain_with_ssl_client_cert_key: >
+        Both ssl_client_cert_chain and ssl_client_cert_key need to be provided.
+      ssl_client_cert_key_password_with_ssl_client_cert_key: >
+        Both ssl_client_cert_key_password and ssl_client_cert_key need to be provided.
+      sasl_oauth_token_provider_respond_to_token: >
+        sasl_oauth_token_provider needs to respond to a #token method.

data/lib/water_drop.rb

@@ -4,16 +4,13 @@
 %w[
   json
   delivery_boy
-  null_logger
   singleton
   dry-configurable
   dry/monitor/notifications
   dry-validation
+  zeitwerk
 ].each { |lib| require lib }
 
-# Internal components
-base_path = File.dirname(__FILE__) + '/water_drop'
-
 # WaterDrop library
 module WaterDrop
   class << self
@@ -44,16 +41,8 @@ module WaterDrop
   end
 end
 
-%w[
-  version
-  instrumentation/monitor
-  instrumentation/listener
-  schemas/message_options
-  schemas/config
-  config
-  config_applier
-  errors
-  base_producer
-  sync_producer
-  async_producer
-].each { |lib| require "#{base_path}/#{lib}" }
+Zeitwerk::Loader
+  .for_gem
+  .tap { |loader| loader.ignore("#{__dir__}/waterdrop.rb") }
+  .tap(&:setup)
+  .tap(&:eager_load)

data/lib/water_drop/async_producer.rb

@@ -19,8 +19,8 @@ module WaterDrop
       d_method = WaterDrop.config.raise_on_buffer_overflow ? :deliver_async! : :deliver_async
 
       DeliveryBoy.send(d_method, message, options)
-    rescue Kafka::Error => error
-      graceful_attempt?(attempts_count, message, options, error) ? retry : raise(error)
+    rescue Kafka::Error => e
+      graceful_attempt?(attempts_count, message, options, e) ? retry : raise(e)
     end
   end
 end

data/lib/water_drop/base_producer.rb

@@ -4,6 +4,12 @@ module WaterDrop
   # Base messages producer that contains all the logic that is exactly the same for both
   # sync and async producers
   class BaseProducer
+    # Contract for checking the correctness of the provided data that someone wants to
+    # dispatch to Kafka
+    SCHEMA = Contracts::MessageOptions.new.freeze
+
+    private_constant :SCHEMA
+
     class << self
       private
 
@@ -12,18 +18,19 @@ module WaterDrop
       # @raise [WaterDrop::Errors::InvalidMessageOptions] raised when message options are
       #   somehow invalid and we cannot perform delivery because of that
       def validate!(options)
-        validation_result = Schemas::MessageOptions.call(options)
+        validation_result = SCHEMA.call(options)
         return true if validation_result.success?
+
         raise Errors::InvalidMessageOptions, validation_result.errors
       end
 
       # Upon failed delivery, we may try to resend a message depending on the attempt number
-      #   or reraise an error if we're unable to do that after given number of retries
+      #   or re-raise an error if we're unable to do that after given number of retries
       #   This method checks that and also instruments errors and retries for the delivery
       # @param attempts_count [Integer] number of attempt (starting from 1) for the delivery
       # @param message [String] message that we want to send to Kafka
       # @param options [Hash] options (including topic) for producer
-      # @param error [Kafka::Error] error that occured
+      # @param error [Kafka::Error] error that occurred
       # @return [Boolean] true if this is a graceful attempt and we can retry or false it this
       #   was the final one and we should deal with the fact, that we cannot deliver a given
       #   message

data/lib/water_drop/config.rb

@@ -3,18 +3,24 @@
 # Configuration and descriptions are based on the delivery boy zendesk gem
 # @see https://github.com/zendesk/delivery_boy
 module WaterDrop
-  # Configurator for setting up all options required by WaterDrop
+  # Configuration object for setting up all options required by WaterDrop
   class Config
     extend Dry::Configurable
 
+    # Config schema definition
+    # @note We use a single instance not to create new one upon each usage
+    SCHEMA = Contracts::Config.new.freeze
+
+    private_constant :SCHEMA
+
     # WaterDrop options
     # option client_id [String] identifier of this producer
     setting :client_id, 'waterdrop'
     # option [Instance, nil] logger that we want to use or nil to fallback to ruby-kafka logger
-    setting :logger, NullLogger.new
+    setting :logger, Logger.new($stdout, level: Logger::WARN)
     # option [Instance] monitor that we want to use. See instrumentation part of the README for
     #   more details
-    setting :monitor, WaterDrop::Instrumentation::Monitor.instance
+    setting :monitor, WaterDrop::Instrumentation::Monitor.new
     # option [Boolean] should we send messages. Setting this to false can be really useful when
     #   testing and or developing because when set to false, won't actually ping Kafka
     setting :deliver, true
@@ -108,10 +114,20 @@ module WaterDrop
       setting :sasl_scram_password, nil
       # option sasl_scram_mechanism [String, nil] Scram mechanism, either 'sha256' or 'sha512'
       setting :sasl_scram_mechanism, nil
+      # option sasl_over_ssl [Boolean] whether to enforce SSL with SASL
+      setting :sasl_over_ssl, true
+      # option ssl_client_cert_chain [String, nil] client cert chain or nil if not used
+      setting :ssl_client_cert_chain, nil
+      # option ssl_client_cert_key_password [String, nil] the password required to read
+      #   the ssl_client_cert_key
+      setting :ssl_client_cert_key_password, nil
+      # @param sasl_oauth_token_provider [Object, nil] OAuthBearer Token Provider instance that
+      #   implements method token.
+      setting :sasl_oauth_token_provider, nil
     end
 
     class << self
-      # Configurating method
+      # Configuration method
       # @yield Runs a block of code providing a config singleton instance to it
       # @yieldparam [WaterDrop::Config] WaterDrop config instance
       def setup
@@ -128,9 +144,10 @@ module WaterDrop
       # @raise [WaterDrop::Errors::InvalidConfiguration] raised when something is wrong with
       #   the configuration
       def validate!(config_hash)
-        validation_result = Schemas::Config.call(config_hash)
+        validation_result = SCHEMA.call(config_hash)
         return true if validation_result.success?
-        raise Errors::InvalidConfiguration, validation_result.errors
+
+        raise Errors::InvalidConfiguration, validation_result.errors.to_h
       end
     end
   end

data/lib/water_drop/config_applier.rb

@@ -29,13 +29,14 @@ module WaterDrop
       # that are ported 1:1 from ruby-kafka
       # For some crazy reason, delivery boy requires compression codec as a string, when
       # ruby-kafka as a symbol. We follow ruby-kafka internal design, so we had to mimic
-      # that by assigning a stringified version that down the road will be symbolized again
+      # that by assigning a string version that down the road will be symbolized again
       # by delivery boy
       # @param delivery_boy_config [DeliveryBoy::Config] delivery boy config instance
       # @param codec_name [Symbol] codec name as a symbol
       def compression_codec(delivery_boy_config, codec_name)
         # If there is no compression codec, we don't apply anything
         return unless codec_name
+
         delivery_boy_config.compression_codec = codec_name.to_s
       end
 

data/lib/water_drop/contracts.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  # Namespace for all the contracts for config validations
+  module Contracts
+    # Regex to check that topic has a valid format
+    TOPIC_REGEXP = /\A(\w|\-|\.)+\z/.freeze
+  end
+end

data/lib/water_drop/contracts/config.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Contracts
+    # Contract with validation rules for WaterDrop configuration details
+    class Config < Dry::Validation::Contract
+      # Valid uri schemas of Kafka broker url
+      URI_SCHEMES = %w[kafka kafka+ssl plaintext ssl].freeze
+
+      # Available sasl scram mechanism of authentication (plus nil)
+      SASL_SCRAM_MECHANISMS = %w[sha256 sha512].freeze
+
+      # Supported compression codecs
+      COMPRESSION_CODECS = %i[snappy gzip lz4 zstd].freeze
+
+      config.messages.load_paths << File.join(WaterDrop.gem_root, 'config', 'errors.yml')
+
+      class << self
+        private
+
+        # Builder for kafka scoped data custom rules
+        # @param keys [Symbol, Hash] the keys names
+        # @param block [Proc] block we want to run with validations within the kafka scope
+        def kafka_scope_rule(*keys, &block)
+          rule(*[:kafka].product(keys)) do
+            instance_exec(values[:kafka], &block)
+          end
+        end
+      end
+
+      private
+
+      # Uri validator to check if uri is in a Kafka acceptable format
+      # @param uri [String] uri we want to validate
+      # @return [Boolean] true if it is a valid uri, otherwise false
+      def broker_schema?(uri)
+        uri = URI.parse(uri)
+        URI_SCHEMES.include?(uri.scheme) && uri.port
+      rescue URI::InvalidURIError
+        false
+      end
+
+      params do
+        required(:client_id).filled(:str?, format?: Contracts::TOPIC_REGEXP)
+        required(:logger).filled
+        required(:deliver).filled(:bool?)
+        required(:raise_on_buffer_overflow).filled(:bool?)
+
+        required(:kafka).schema do
+          required(:seed_brokers).value(:array, :filled?).each(:str?)
+          required(:connect_timeout).filled(:int?, gt?: 0)
+          required(:socket_timeout).filled(:int?, gt?: 0)
+          required(:compression_threshold).filled(:int?, gteq?: 1)
+          optional(:compression_codec).maybe(included_in?: COMPRESSION_CODECS)
+
+          required(:max_buffer_bytesize).filled(:int?, gt?: 0)
+          required(:max_buffer_size).filled(:int?, gt?: 0)
+          required(:max_queue_size).filled(:int?, gt?: 0)
+
+          required(:ack_timeout).filled(:int?, gt?: 0)
+          required(:delivery_interval).filled(:int?, gteq?: 0)
+          required(:delivery_threshold).filled(:int?, gteq?: 0)
+
+          required(:max_retries).filled(:int?, gteq?: 0)
+          required(:retry_backoff).filled(:int?, gteq?: 0)
+          required(:required_acks).filled(included_in?: [1, 0, -1, :all])
+
+          %i[
+            ssl_ca_cert
+            ssl_ca_cert_file_path
+            ssl_client_cert
+            ssl_client_cert_key
+            ssl_client_cert_chain
+            ssl_client_cert_key_password
+            sasl_gssapi_principal
+            sasl_gssapi_keytab
+            sasl_plain_authzid
+            sasl_plain_username
+            sasl_plain_password
+            sasl_scram_username
+            sasl_scram_password
+          ].each do |encryption_attribute|
+            optional(encryption_attribute).maybe(:str?)
+          end
+
+          optional(:ssl_ca_certs_from_system).maybe(:bool?)
+          optional(:sasl_over_ssl).maybe(:bool?)
+          optional(:sasl_oauth_token_provider).value(:any)
+
+          # It's not with other encryptions as it has some more rules
+          optional(:sasl_scram_mechanism)
+            .maybe(:str?, included_in?: SASL_SCRAM_MECHANISMS)
+        end
+      end
+
+      kafka_scope_rule(:seed_brokers) do |kafka|
+        unless kafka[:seed_brokers].all?(&method(:broker_schema?))
+          key(%i[kafka seed_brokers]).failure(:broker_schema)
+        end
+      end
+
+      kafka_scope_rule(:ssl_client_cert, :ssl_client_cert_key) do |kafka|
+        if kafka[:ssl_client_cert] &&
+           kafka[:ssl_client_cert_key].nil?
+          key(%i[kafka ssl_client_cert_key]).failure(:ssl_client_cert_with_ssl_client_cert_key)
+        end
+      end
+
+      kafka_scope_rule(:ssl_client_cert_key, :ssl_client_cert) do |kafka|
+        if kafka[:ssl_client_cert_key] &&
+           kafka[:ssl_client_cert].nil?
+          key.failure(:ssl_client_cert_key_with_ssl_client_cert)
+        end
+      end
+
+      kafka_scope_rule(:ssl_client_cert_chain, :ssl_client_cert) do |kafka|
+        if kafka[:ssl_client_cert_chain] &&
+           kafka[:ssl_client_cert].nil?
+          key.failure(:ssl_client_cert_chain_with_ssl_client_cert)
+        end
+      end
+
+      kafka_scope_rule(:ssl_client_cert_key_password, :ssl_client_cert_key) do |kafka|
+        if kafka[:ssl_client_cert_key_password] &&
+           kafka[:ssl_client_cert_key].nil?
+          key.failure(:ssl_client_cert_key_password_with_ssl_client_cert_key)
+        end
+      end
+
+      kafka_scope_rule(:sasl_oauth_token_provider) do |kafka|
+        if kafka[:sasl_oauth_token_provider] &&
+           !kafka[:sasl_oauth_token_provider].respond_to?(:token)
+          key.failure(:sasl_oauth_token_provider_respond_to_token)
+        end
+      end
+    end
+  end
+end

data/lib/water_drop/contracts/message_options.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Contracts
+    # Contract with validation rules for validating that all the message options that
+    # we provide to producer ale valid and usable
+    # @note Does not validate message itself as it is not our concern
+    class MessageOptions < Dry::Validation::Contract
+      params do
+        required(:topic).filled(:str?, format?: TOPIC_REGEXP)
+        optional(:key).maybe(:str?, :filled?)
+        optional(:partition).filled(:int?, gteq?: 0)
+        optional(:partition_key).maybe(:str?, :filled?)
+        optional(:create_time).maybe(:time?)
+        optional(:headers).maybe(:hash?)
+      end
+    end
+  end
+end

data/lib/water_drop/errors.rb

@@ -6,10 +6,10 @@ module WaterDrop
     # Base class for all the WaterDrop internal errors
     BaseError = Class.new(StandardError)
 
-    # Raised when configuration doesn't match with validation schema
+    # Raised when configuration doesn't match with validation contract
     InvalidConfiguration = Class.new(BaseError)
 
-    # Raised when we try to send message with invalid optionss
+    # Raised when we try to send message with invalid options
     InvalidMessageOptions = Class.new(BaseError)
 
     # Raised when want to hook up to an event that is not registered and supported

data/lib/water_drop/instrumentation/monitor.rb

@@ -5,13 +5,11 @@ module WaterDrop
   module Instrumentation
     # Monitor is used to hookup external monitoring services to monitor how WaterDrop works
     # Since it is a pub-sub based on dry-monitor, you can use as many subscribers/loggers at the
-    # same time, which means that you might have for example file logging and newrelic at the same
+    # same time, which means that you might have for example file logging and NewRelic at the same
     # time
     # @note This class acts as a singleton because we are only permitted to have single monitor
     #   per running process (just as logger)
     class Monitor < Dry::Monitor::Notifications
-      include Singleton
-
       # List of events that we support in the system and to which a monitor client can hook up
       # @note The non-error once support timestamp benchmarking
       BASE_EVENTS = %w[
@@ -35,6 +33,7 @@ module WaterDrop
       def subscribe(event_name_or_listener)
         return super unless event_name_or_listener.is_a?(String)
         return super if available_events.include?(event_name_or_listener)
+
         raise Errors::UnregisteredMonitorEvent, event_name_or_listener
       end
 

data/lib/water_drop/instrumentation/{listener.rb → stdout_listener.rb}

@@ -6,7 +6,7 @@ module WaterDrop
     # It can be removed/replaced or anything without any harm to the Waterdrop flow
     # @note It is a module as we can use it then as a part of the Karafka framework listener
     #   as well as we can use it standalone
-    module Listener
+    class StdoutListener
       # Log levels that we use in this particular listener
       USED_LOG_LEVELS = %i[
         info
@@ -33,20 +33,12 @@ module WaterDrop
 
           info "Attempt #{attempts_count} of delivery to: #{options} because of #{error}"
         end
-
-        module_function error_name
-        module_function retry_name
-        public error_name
-        public retry_name
       end
 
       USED_LOG_LEVELS.each do |log_level|
         define_method log_level do |*args|
           WaterDrop.logger.send(log_level, *args)
         end
-
-        module_function log_level
-        private_class_method log_level
       end
     end
   end

data/lib/water_drop/sync_producer.rb

@@ -17,8 +17,8 @@ module WaterDrop
       return unless WaterDrop.config.deliver
 
       DeliveryBoy.deliver(message, options)
-    rescue Kafka::Error => error
-      graceful_attempt?(attempts_count, message, options, error) ? retry : raise(error)
+    rescue Kafka::Error => e
+      graceful_attempt?(attempts_count, message, options, e) ? retry : raise(e)
     end
   end
 end

data/lib/water_drop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '1.2.5'
+  VERSION = '1.3.0.rc1'
 end

data/waterdrop.gemspec

@@ -19,16 +19,17 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'delivery_boy', '~> 0.2'
   spec.add_dependency 'dry-configurable', '~> 0.8'
   spec.add_dependency 'dry-monitor', '~> 0.3'
-  spec.add_dependency 'dry-validation', '~> 0.11'
-  spec.add_dependency 'null-logger', '~> 0.1'
-  spec.add_dependency 'ruby-kafka', '>= 0.6'
+  spec.add_dependency 'dry-validation', '~> 1.2'
+  spec.add_dependency 'ruby-kafka', '>= 0.7.8'
+  spec.add_dependency 'zeitwerk', '~> 2.1'
 
-  spec.post_install_message = <<~MSG
-    \e[93mWarning:\e[0m If you're using Kafka 0.10, please lock ruby-kafka in your Gemfile to version '0.6.8':
-    gem 'ruby-kafka', '~> 0.6.8'
-  MSG
+  spec.required_ruby_version = '>= 2.4.0'
 
-  spec.required_ruby_version = '>= 2.3.0'
+  spec.cert_chain  = ['certs/mensfeld.pem']
+
+  if $PROGRAM_NAME.end_with?('gem')
+    spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
+  end
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }

metadata

@@ -1,14 +1,40 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 1.2.5
+  version: 1.3.0.rc1
 platform: ruby
 authors:
 - Maciej Mensfeld
 autorequire: 
 bindir: bin
-cert_chain: []
-date: 2019-02-05 00:00:00.000000000 Z
+cert_chain:
+- |
+  -----BEGIN CERTIFICATE-----
+  MIIEODCCAqCgAwIBAgIBATANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhtYWNp
+  ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMTkwNzMwMTQ1NDU0WhcNMjAwNzI5MTQ1
+  NDU0WjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
+  CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQC9fCwtaHZG2SyyNXiH8r0QbJQx/xxl
+  dkvwWz9QGJO+O8rEx20FB1Ab+MVkfOscwIv5jWpmk1U9whzDPl1uFtIbgu+sk+Zb
+  uQlZyK/DPN6c+/BbBL+RryTBRyvkPLoCVwm7uxc/JZ1n4AI6eF4cCZ2ieZ9QgQbU
+  MQs2QPqs9hT50Ez/40GnOdadVfiDDGz+NME2C4ms0BriXwZ1tcRTfJIHe2xjIbbb
+  y5qRGfsLKcgMzvLQR24olixyX1MR0s4+Wveq3QL/gBhL4veUcv+UABJA8IJR0kyB
+  seHHutusiwZ1v3SjjjW1xLLrc2ARV0mgCb0WaK2T4iA3oFTGLh6Ydz8LNl31KQFv
+  94nRd8IhmJxrhQ6dQ/WT9IXoa5S9lfT5lPJeINemH4/6QPABzf9W2IZlCdI9wCdB
+  TBaw57MKneGAYZiKjw6OALSy2ltQUCl3RqFl3VP7n8uFy1U987Q5VIIQ3O1UUsQD
+  Oe/h+r7GUU4RSPKgPlrwvW9bD/UQ+zF51v8CAwEAAaN3MHUwCQYDVR0TBAIwADAL
+  BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFJNIBHdfEUD7TqHqIer2YhWaWhwcMB0GA1Ud
+  EQQWMBSBEm1hY2llakBtZW5zZmVsZC5wbDAdBgNVHRIEFjAUgRJtYWNpZWpAbWVu
+  c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBAKA4eqko6BTNhlysip6rfBkVTGri
+  ZXsL+kRb2hLvsQJS/kLyM21oMlu+LN0aPj3qEFR8mE/YeDD8rLAfruBRTltPNbR7
+  xA5eE1gkxY5LfExUtK3b2wPqfmo7mZgfcsMwfYg/tUXw1WpBCnrhAJodpGH6SXmp
+  A40qFUZst0vjiOoO+aTblIHPmMJXoZ3K42dTlNKlEiDKUWMRKSgpjjYGEYalFNWI
+  hHfCz2r8L2t+dYdMZg1JGbEkq4ADGsAA8ioZIpJd7V4hI17u5TCdi7X5wh/0gN0E
+  CgP+nLox3D+l2q0QuQEkayr+auFYkzTCkF+BmEk1D0Ru4mcf3F4CJvEmW4Pzbjqt
+  i1tsCWPtJ4E/UUKnKaWKqGbjrjHJ0MuShYzHkodox5IOiCXIQg+1+YSzfXUV6WEK
+  KJG/fhg1JV5vVDdVy6x+tv5SQ5ctU0feCsVfESi3rE3zRd+nvzE9HcZ5aXeL1UtJ
+  nT5Xrioegu2w1jPyVEgyZgTZC5rvD0nNS5sFNQ==
+  -----END CERTIFICATE-----
+date: 2019-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: delivery_boy
@@ -58,42 +84,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.11'
+        version: '1.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.11'
+        version: '1.2'
 - !ruby/object:Gem::Dependency
-  name: null-logger
+  name: ruby-kafka
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: 0.7.8
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: 0.7.8
 - !ruby/object:Gem::Dependency
-  name: ruby-kafka
+  name: zeitwerk
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '2.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '2.1'
 description: Kafka messaging made easy!
 email:
 - maciej@mensfeld.pl
@@ -101,7 +127,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".coditsu.yml"
+- ".coditsu/ci.yml"
+- ".github/FUNDING.yml"
 - ".gitignore"
 - ".rspec"
 - ".ruby-gemset"
@@ -112,27 +139,29 @@ files:
 - Gemfile.lock
 - MIT-LICENCE
 - README.md
+- certs/mensfeld.pem
 - config/errors.yml
 - lib/water_drop.rb
 - lib/water_drop/async_producer.rb
 - lib/water_drop/base_producer.rb
 - lib/water_drop/config.rb
 - lib/water_drop/config_applier.rb
+- lib/water_drop/contracts.rb
+- lib/water_drop/contracts/config.rb
+- lib/water_drop/contracts/message_options.rb
 - lib/water_drop/errors.rb
-- lib/water_drop/instrumentation/listener.rb
 - lib/water_drop/instrumentation/monitor.rb
-- lib/water_drop/schemas/config.rb
-- lib/water_drop/schemas/message_options.rb
+- lib/water_drop/instrumentation/stdout_listener.rb
 - lib/water_drop/sync_producer.rb
 - lib/water_drop/version.rb
 - lib/waterdrop.rb
+- log/.keep
 - waterdrop.gemspec
 homepage: https://github.com/karafka/waterdrop
 licenses:
 - MIT
 metadata: {}
-post_install_message: "\e[93mWarning:\e[0m If you're using Kafka 0.10, please lock
-  ruby-kafka in your Gemfile to version '0.6.8':\ngem 'ruby-kafka', '~> 0.6.8'\n"
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -140,14 +169,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.3.0
+      version: 2.4.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubygems_version: 3.0.1
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Kafka messaging made easy!

data/lib/water_drop/schemas/config.rb

@@ -1,81 +0,0 @@
-# frozen_string_literal: true
-
-module WaterDrop
-  # Namespace for all the schemas for config validations
-  module Schemas
-    # Schema with validation rules for WaterDrop configuration details
-    Config = Dry::Validation.Schema do
-      # Valid uri schemas of Kafka broker url
-      URI_SCHEMES = %w[
-        kafka
-        kafka+ssl
-      ].freeze
-
-      # Available sasl scram mechanism of authentication (plus nil)
-      SASL_SCRAM_MECHANISMS ||= %w[sha256 sha512].freeze
-
-      configure do
-        config.messages_file = File.join(
-          WaterDrop.gem_root, 'config', 'errors.yml'
-        )
-
-        # Uri validator to check if uri is in a Kafka acceptable format
-        # @param uri [String] uri we want to validate
-        # @return [Boolean] true if it is a valid uri, otherwise false
-        def broker_schema?(uri)
-          uri = URI.parse(uri)
-          URI_SCHEMES.include?(uri.scheme) && uri.port
-        rescue URI::InvalidURIError
-          false
-        end
-      end
-
-      required(:client_id).filled(:str?, format?: Schemas::TOPIC_REGEXP)
-      required(:logger).filled
-      required(:deliver).filled(:bool?)
-      required(:raise_on_buffer_overflow).filled(:bool?)
-
-      required(:kafka).schema do
-        required(:seed_brokers).filled { each(:broker_schema?) }
-        required(:connect_timeout).filled(:int?, gt?: 0)
-        required(:socket_timeout).filled(:int?, gt?: 0)
-        required(:compression_threshold).filled(:int?, gteq?: 1)
-        optional(:compression_codec).maybe(included_in?: %i[snappy gzip lz4])
-
-        required(:max_buffer_bytesize).filled(:int?, gt?: 0)
-        required(:max_buffer_size).filled(:int?, gt?: 0)
-        required(:max_queue_size).filled(:int?, gt?: 0)
-
-        required(:ack_timeout).filled(:int?, gt?: 0)
-        required(:delivery_interval).filled(:int?, gteq?: 0)
-        required(:delivery_threshold).filled(:int?, gteq?: 0)
-
-        required(:max_retries).filled(:int?, gteq?: 0)
-        required(:retry_backoff).filled(:int?, gteq?: 0)
-        required(:required_acks).filled(included_in?: [1, 0, -1, :all])
-
-        %i[
-          ssl_ca_cert
-          ssl_ca_cert_file_path
-          ssl_client_cert
-          ssl_client_cert_key
-          sasl_gssapi_principal
-          sasl_gssapi_keytab
-          sasl_plain_authzid
-          sasl_plain_username
-          sasl_plain_password
-          sasl_scram_username
-          sasl_scram_password
-        ].each do |encryption_attribute|
-          optional(encryption_attribute).maybe(:str?)
-        end
-
-        optional(:ssl_ca_certs_from_system).maybe(:bool?)
-
-        # It's not with other encryptions as it has some more rules
-        optional(:sasl_scram_mechanism)
-          .maybe(:str?, included_in?: WaterDrop::Schemas::SASL_SCRAM_MECHANISMS)
-      end
-    end
-  end
-end

data/lib/water_drop/schemas/message_options.rb

@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-module WaterDrop
-  module Schemas
-    # Regexp to check that topic has a valid format
-    TOPIC_REGEXP = /\A(\w|\-|\.)+\z/
-
-    # Schema with validation rules for validating that all the message options that
-    # we provide to producer ale valid and usable
-    # @note Does not validate message itself as it is not our concern
-    MessageOptions = Dry::Validation.Schema do
-      required(:topic).filled(:str?, format?: TOPIC_REGEXP)
-      optional(:key).maybe(:str?, :filled?)
-      optional(:partition).filled(:int?, gteq?: 0)
-      optional(:partition_key).maybe(:str?, :filled?)
-      optional(:create_time).maybe(:time?)
-    end
-  end
-end