waterdrop 0.4.0 → 1.0.0.alpha1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d6ebd8fb5186e625ad83a4039b1953e145e6ed6e
-  data.tar.gz: 3f5b6e5d6ac383388a4fff4d18f6600122e91bbc
+  metadata.gz: 93db4ab2979f93cc621a72ee8792a6e7f7f6c1f0
+  data.tar.gz: acbeb2ccbb8eee1e53b47e1b9cb4da1d484e6b19
 SHA512:
-  metadata.gz: 2ffc2c7b34e2e9f53b61db43de61fe5f557af524ffba98f1101a1a630d5af021d90d76e0d65a0c06a2d13114933240d9f984d0b359251ac58af84e2afff04320
-  data.tar.gz: d1617c0169e356b16332d9b70285e58f0b524185ca2cc413903d53316f379bf1a58a1d383c990883cb2eb827194a6c9d2959d3459a3f04f99548c3c2d5394f47
+  metadata.gz: b0abf19f2a2fd07db54d472c7a8f16c32205688377d3ae6038d5dc7d19ece08d3bbdde42e6031647dd914b3e4ccee612929992a4ee991cc3f020c93702e8e365
+  data.tar.gz: d177d752b481ba9fde09cd63edc6793e735cdf5516e28071bfa553269acb6fd3b2acace3cea22a870f5885eccb9f08006745246f39a49a065e6e92b1ff33c1b1

data/.ruby-version

@@ -1 +1 @@
-2.4.0
+2.4.2

data/.travis.yml

@@ -1,9 +1,17 @@
 language: ruby
+sudo: false
 rvm:
   - 2.3.0
   - 2.3.1
   - 2.3.2
   - 2.3.3
+  - 2.3.4
   - 2.4.0
-script:
-  - bundle exec rake
+  - 2.4.1
+  - 2.4.2
+  - jruby-head
+script: bundle exec rspec spec/
+env:
+  global:
+    - JRUBY_OPTS='--debug'
+install: bundle install --jobs=3 --retry=3

data/CHANGELOG.md

@@ -1,6 +1,18 @@
 # WaterDrop changelog
 
-## 0.4 unreleased
+## 1.0.0 unreleased
+
+- #37 - ack level for producer
+- Gem bump
+- Ruby 2.4.2 support
+- Raw ruby-kafka driver is now replaced with delivery_boy
+- Sync and async producers
+- Complete update of the API
+- Much better validations for config details
+- Complete API remodel - please read the new README
+- Renamed send_messages to deliver
+
+## 0.4
 - Bump to match Karafka
 - Renamed ```hosts``` to ```seed_brokers```
 - Removed the ```ssl``` scoping for ```kafka``` config namespace to better match Karafka conventions

data/Gemfile

@@ -1,4 +1,10 @@
 # frozen_string_literal: true
 
 source 'https://rubygems.org'
+
 gemspec
+
+group :development, :test do
+  gem 'rspec'
+  gem 'simplecov'
+end

data/Gemfile.lock

@@ -1,57 +1,79 @@
 PATH
   remote: .
   specs:
-    waterdrop (0.4.0)
-      bundler
-      connection_pool
-      dry-configurable (~> 0.6)
+    waterdrop (1.0.0.alpha1)
+      delivery_boy (>= 0.2.2)
+      dry-configurable (~> 0.7)
+      dry-validation (~> 0.11)
       null-logger
-      rake
-      ruby-kafka (~> 0.4)
 
 GEM
   remote: https://rubygems.org/
   specs:
     concurrent-ruby (1.0.5)
-    connection_pool (2.2.1)
+    delivery_boy (0.2.2)
+      king_konf (~> 0.1.8)
+      ruby-kafka (~> 0.4)
     diff-lcs (1.3)
     docile (1.1.5)
     dry-configurable (0.7.0)
       concurrent-ruby (~> 1.0)
-    ffi (1.9.18)
-    gssapi (1.2.0)
-      ffi (>= 1.0.1)
+    dry-container (0.6.0)
+      concurrent-ruby (~> 1.0)
+      dry-configurable (~> 0.1, >= 0.1.3)
+    dry-core (0.3.4)
+      concurrent-ruby (~> 1.0)
+    dry-equalizer (0.2.0)
+    dry-logic (0.4.2)
+      dry-container (~> 0.2, >= 0.2.6)
+      dry-core (~> 0.2)
+      dry-equalizer (~> 0.2)
+    dry-types (0.12.1)
+      concurrent-ruby (~> 1.0)
+      dry-configurable (~> 0.1)
+      dry-container (~> 0.3)
+      dry-core (~> 0.2, >= 0.2.1)
+      dry-equalizer (~> 0.2)
+      dry-logic (~> 0.4, >= 0.4.2)
+      inflecto (~> 0.0.0, >= 0.0.2)
+    dry-validation (0.11.1)
+      concurrent-ruby (~> 1.0)
+      dry-configurable (~> 0.1, >= 0.1.3)
+      dry-core (~> 0.2, >= 0.2.1)
+      dry-equalizer (~> 0.2)
+      dry-logic (~> 0.4, >= 0.4.0)
+      dry-types (~> 0.12.0)
+    inflecto (0.0.2)
     json (2.1.0)
+    king_konf (0.1.8)
     null-logger (0.1.4)
-    rake (12.0.0)
-    rspec (3.6.0)
-      rspec-core (~> 3.6.0)
-      rspec-expectations (~> 3.6.0)
-      rspec-mocks (~> 3.6.0)
-    rspec-core (3.6.0)
-      rspec-support (~> 3.6.0)
-    rspec-expectations (3.6.0)
+    rspec (3.7.0)
+      rspec-core (~> 3.7.0)
+      rspec-expectations (~> 3.7.0)
+      rspec-mocks (~> 3.7.0)
+    rspec-core (3.7.0)
+      rspec-support (~> 3.7.0)
+    rspec-expectations (3.7.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.6.0)
-    rspec-mocks (3.6.0)
+      rspec-support (~> 3.7.0)
+    rspec-mocks (3.7.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.6.0)
-    rspec-support (3.6.0)
-    ruby-kafka (0.4.0)
-      gssapi (>= 1.2.0)
-    simplecov (0.14.1)
+      rspec-support (~> 3.7.0)
+    rspec-support (3.7.0)
+    ruby-kafka (0.5.0)
+    simplecov (0.15.1)
       docile (~> 1.1.0)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
-    simplecov-html (0.10.1)
+    simplecov-html (0.10.2)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  rspec (~> 3.6.0)
-  simplecov (~> 0.14.1)
+  rspec
+  simplecov
   waterdrop!
 
 BUNDLED WITH
-   1.13.7
+   1.15.4

data/README.md

@@ -1,11 +1,16 @@
 # WaterDrop
 
 [![Build Status](https://travis-ci.org/karafka/waterdrop.png)](https://travis-ci.org/karafka/waterdrop)
-[![Code Climate](https://codeclimate.com/github/karafka/waterdrop/badges/gpa.svg)](https://codeclimate.com/github/karafka/waterdrop)
-[![Gem Version](https://badge.fury.io/rb/waterdrop.svg)](http://badge.fury.io/rb/waterdrop)
 [![Join the chat at https://gitter.im/karafka/karafka](https://badges.gitter.im/karafka/karafka.svg)](https://gitter.im/karafka/karafka?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
-Gem used to send messages to Kafka in an easy way.
+Gem used to send messages to Kafka in an easy way with an extra validation layer. It is a part of the [Karafka](https://github.com/karafka/karafka) ecosystem.
+
+WaterDrop is based on Zendesks [delivery_boy](https://github.com/zendesk/delivery_boy) gem.
+
+It is:
+
+ - Thread safe
+ - Supports sync and async producers
 
 ## Installation
 
@@ -27,97 +32,103 @@ bundle install
 
 ## Setup
 
-WaterDrop has following configuration options:
-
-| Option                      | Required   | Value type    | Description                                                                        |
-|-----------------------------|------------|---------------|------------------------------------------------------------------------------------|
-| send_messages               | true       | Boolean       | Should we send messages to Kafka                                                   |
-| connect_timeout             | false      | Integer       | Number of seconds to wait while connecting to a broker for the first time          |
-| socket_timeout              | false      | Integer       | Number of seconds to wait when reading from or writing to a socket                 |
-| connection_pool.size        | true       | Integer       | Kafka connection pool size                                                         |
-| connection_pool.timeout     | true       | Integer       | Kafka connection pool timeout                                                      |
-| kafka.seed_brokers          | true       | Array<String> | Kafka servers hosts with ports                                                     |
-| raise_on_failure            | true       | Boolean       | Should we raise an exception when we cannot send message to Kafka - if false will silently ignore failures |
-| kafka.ssl_ca_cert           | false      | String        | SSL CA certificate                                                                 |
-| kafka.ssl_ca_cert_file_path | false      | String        | SSL CA certificate file path                                                       |
-| kafka.ssl_client_cert       | false      | String        | SSL client certificate                                                             |
-| kafka.ssl_client_cert_key   | false      | String        | SSL client certificate password                                                    |
-| kafka.sasl_gssapi_principal | false      | String        | SASL principal                                                                     |
-| kafka.sasl_gssapi_keytab    | false      | String        | SASL keytab                                                                        |
-| kafka.sasl_plain_authzid    | false      | String        | The authorization identity to use                                                  |
-| kafka.sasl_plain_username   | false      | String        | The username used to authenticate                                                  |
-| kafka.sasl_plain_password   | false      | String        | The password used to authenticate                                                  |
-
-To apply this configuration, you need to use a *setup* method:
+WaterDrop is a complex tool, that contains multiple configuration options. To keep everything organized, all the configuration options were divided into two groups:
+
+- WaterDrop options - options directly related to Karafka framework and it's components
+- Ruby-Kafka driver options - options related to Ruby-Kafka/Delivery boy
+
+To apply all those configuration options, you need to use the ```#setup``` method:
 
 ```ruby
 WaterDrop.setup do |config|
-  config.send_messages = true
-  config.connection_pool.size = 20
-  config.connection_pool.timeout = 1
-  config.kafka.seed_brokers = ['localhost:9092']
-  config.raise_on_failure = true
+  config.deliver = true
+  config.kafka.seed_brokers = %w[kafka://localhost:9092]
 end
 ```
 
-This configuration can be placed in *config/initializers* and can vary based on the environment:
+### WaterDrop configuration options
+
+| Option                      | Description                                                      |
+|-----------------------------|------------------------------------------------------------------|
+| client_id                   | This is how the client will identify itself to the Kafka brokers |
+| logger                      | Logger that we want to use                                       |
+| deliver                     | Should we send messages to Kafka                                 |
+
+### Ruby-Kafka driver and Delivery boy configuration options
+
+**Note:** We've listed here only **the most important** configuration options. If you're interested in all the options, please go to the [config.rb](https://github.com/karafka/waterdrop/blob/master/lib/water_drop/config.rb) file for more details.
+
+**Note:** All the options are subject to validations. In order to check what is and what is not acceptable, please go to the [config.rb validation schema](https://github.com/karafka/waterdrop/blob/master/lib/water_drop/schemas/config.rb) file.
+
+| Option              | Description                                                                                                                                           |
+|---------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------|
+| delivery_interval   | The number of seconds between background message deliveries. Disable timer-based background deliveries by setting this to 0.                          |
+| delivery_threshold  | The number of buffered messages that will trigger a background message delivery. Disable buffer size based background deliveries by setting this to 0.|
+| required_acks       | The number of Kafka replicas that must acknowledge messages before they're considered as successfully written.                                        |
+| ack_timeout         | A timeout executed by a broker when the client is sending messages to it.                                                                             |
+| max_retries         | The number of retries when attempting to deliver messages.                                                                                            |
+| retry_backoff       | The number of seconds to wait after a failed attempt to send messages to a Kafka broker before retrying.                                              |
+| max_buffer_bytesize | The maximum number of bytes allowed in the buffer before new messages are rejected.                                                                   |
+| max_buffer_size     | The maximum number of messages allowed in the buffer before new messages are rejected.                                                                |
+| max_queue_size      | The maximum number of messages allowed in the queue before new messages are rejected.                                                                 |
+| sasl_plain_username | The username used to authenticate.                                                                                                                    |
+| sasl_plain_password | The password used to authenticate.                                                                                                                    |
+
+This configuration can be also placed in *config/initializers* and can vary based on the environment:
 
 ```ruby
 WaterDrop.setup do |config|
-  config.send_messages = Rails.env.production?
-  config.connection_pool.size = 20
-  config.connection_pool.timeout = 1
-  config.kafka.seed_brokers = [Rails.env.production? ? 'prod-host:9091' : 'localhost:9092']
-  config.raise_on_failure = Rails.env.production?
+  config.deliver = Rails.env.production?
+  config.kafka.seed_brokers = [Rails.env.production? ? 'kafka://prod-host:9091' : 'kafka://localhost:9092']
 end
 ```
 
 ## Usage
 
-### Creating and sending standard messages
-
-To send Kafka messages, just create and send messages directly:
+To send Kafka messages, just use one of the producers:
 
 ```ruby
-message = WaterDrop::Message.new('topic', 'message')
-message.send!
+WaterDrop::SyncProducer.call('message', topic: 'my-topic')
+
+# or for async
 
-message = WaterDrop::Message.new('topic', { user_id: 1 }.to_json)
-message.send!
+WaterDrop::AsyncProducer.call('message', topic: 'my-topic')
 ```
 
-message that you want to send should be either binary or stringified (to_s, to_json, etc).
+Both ```SyncProducer``` and ```AsyncProducer``` accept following options:
 
-### Using aspects to handle messages
+| Option              | Required | Value type     | Description                                                         |
+|-------------------- |----------|----------------|---------------------------------------------------------------------|
+| ```topic```         | true     | String, Symbol | The Kafka topic that should be written to                           |
+| ```key```           | false    | String         | The key that should be set on the Kafka message                     |
+| ```partition```     | false    | Integer        | A specific partition number that should be written to               |
+| ```partition_key``` | false    | String         | A string that can be used to deterministically select the partition |
 
-WaterDrop no longer depends on [Aspector](https://github.com/gcao/aspector). Please refer to [Aspector](https://github.com/gcao/aspector) documentation if you want to handle messaging in an aspect way.
+Keep in mind, that message you want to send should be either binary or stringified (to_s, to_json, etc).
 
 ## References
 
 * [Karafka framework](https://github.com/karafka/karafka)
-* [Waterdrop](https://github.com/karafka/waterdrop)
-* [Worker Glass](https://github.com/karafka/worker-glass)
-* [Envlogic](https://github.com/karafka/envlogic)
-* [Null Logger](https://github.com/karafka/null-logger)
-* [Aspector](https://github.com/gcao/aspector)
 * [WaterDrop Travis CI](https://travis-ci.org/karafka/waterdrop)
-* [WaterDrop Code Climate](https://codeclimate.com/github/karafka/waterdrop)
+* [WaterDrop Coditsu](https://app.coditsu.io/karafka/repositories/waterdrop)
 
 ## Note on Patches/Pull Requests
 
 Fork the project.
 Make your feature addition or bug fix.
-Add tests for it. This is important so I don't break it in a future version unintentionally.
-Commit, do not mess with Rakefile, version, or history. (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull). Send me a pull request. Bonus points for topic branches.
+Add tests for it. This is important so we don't break it in a future versions unintentionally.
+Commit, do not mess with version, or history. (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull). Send me a pull request. Bonus points for topic branches.
+
+[![coditsu](https://coditsu.io/assets/quality_bar.svg)](https://app.coditsu.io/karafka/repositories/waterdrop)
 
-[![coditsu](https://coditsu.io/assets/quality_bar.svg)](https://coditsu.io)
+Each pull request must pass our quality requirements. To check if everything is as it should be, we use [Coditsu](https://coditsu.io) that combines multiple linters and code analyzers for both code and documentation.
 
-Each pull request must pass our quality requirements. To check if everything is as it should be, we use [Coditsu](https://coditsu.io) that combinse multiple linters and code analyzers. Unfortunately, for now it is invite-only based, so just ping us and we will give you access to the quality results.
+Unfortunately, it does not yet support independent forks, however you should be fine by looking at what we require.
 
 Please run:
 
 ```bash
-bundle exec rake
+bundle exec rspec
 ```
 
 to check if everything is in order. After that you can submit a pull request.

data/config/errors.yml

@@ -0,0 +1,6 @@
+en:
+  errors:
+    broker_schema?: >
+      has an invalid format.
+      Expected schema, host and port number.
+      Example: kafka://127.0.0.1:9092 or kafka+ssl://127.0.0.1:9092

data/lib/water_drop.rb

@@ -2,49 +2,62 @@
 
 # External components
 %w[
-  rake
-  rubygems
-  bundler
-  logger
-  pathname
   json
-  kafka
-  forwardable
-  connection_pool
+  delivery_boy
   null_logger
   dry-configurable
+  dry-validation
 ].each { |lib| require lib }
 
 # Internal components
 base_path = File.dirname(__FILE__) + '/water_drop'
 
-%w[
-  version
-  producer_proxy
-  pool
-  config
-  message
-].each { |lib| require "#{base_path}/#{lib}" }
-
 # WaterDrop library
 module WaterDrop
   class << self
-    attr_writer :logger
-
-    # @return [Logger] logger that we want to use
-    def logger
-      @logger ||= NullLogger.new
-    end
+    attr_accessor :logger
 
     # Sets up the whole configuration
     # @param [Block] block configuration block
     def setup(&block)
       Config.setup(&block)
+
+      DeliveryBoy.logger = self.logger = config.logger
+
+      # Recursive lambda for mapping config down to delivery boy
+      applier = lambda { |db_config, h|
+        h.each do |k, v|
+          applier.call(db_config, v) && next if v.is_a?(Hash)
+          next unless db_config.respond_to?(:"#{k}=")
+          db_config.public_send(:"#{k}=", v)
+        end
+      }
+
+      DeliveryBoy.config.tap do |config|
+        config.brokers = Config.config.kafka.seed_brokers
+        applier.call(config, Config.config.to_h)
+      end
     end
 
     # @return [WaterDrop::Config] config instance
     def config
       Config.config
     end
+
+    # @return [String] root path of this gem
+    def gem_root
+      Pathname.new(File.expand_path('../..', __FILE__))
+    end
   end
 end
+
+%w[
+  version
+  schemas/message_options
+  schemas/config
+  config
+  errors
+  base_producer
+  sync_producer
+  async_producer
+].each { |lib| require "#{base_path}/#{lib}" }

data/lib/water_drop/async_producer.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+# WaterDrop library
+module WaterDrop
+  # Async producer for messages
+  AsyncProducer = Class.new(BaseProducer)
+  AsyncProducer.method_name = :deliver_async
+end

data/lib/water_drop/base_producer.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  # Base messages producer that contains all the logic that is exactly the same for both
+  # sync and async producers
+  class BaseProducer
+    class << self
+      # Delivery boy method name that we use to invoke producer action
+      attr_accessor :method_name
+
+      # Performs message delivery using method_name method
+      # @param message [String] message that we want to send to Kafka
+      # @param options [Hash] options (including topic) for producer
+      # @raise [WaterDrop::Errors::InvalidMessageOptions] raised when message options are
+      #   somehow invalid and we cannot perform delivery because of that
+      def call(message, options)
+        validate!(options)
+        return unless WaterDrop.config.deliver
+        DeliveryBoy.public_send(method_name, message, options)
+      end
+
+      private
+
+      # Runs the message options validations and raises an error if anything is invalid
+      # @param options [Hash] hash that we want to validate
+      # @raise [WaterDrop::Errors::InvalidMessageOptions] raised when message options are
+      #   somehow invalid and we cannot perform delivery because of that
+      def validate!(options)
+        validation_result = Schemas::MessageOptions.call(options)
+        return true if validation_result.success?
+        raise Errors::InvalidMessageOptions, validation_result.errors
+      end
+    end
+  end
+end

data/lib/water_drop/config.rb

@@ -1,35 +1,27 @@
 # frozen_string_literal: true
 
+# Configuration and descriptions are based on the delivery boy zendesk gem
+# @see https://github.com/zendesk/delivery_boy
 module WaterDrop
   # Configurator for setting up all options required by WaterDrop
   class Config
     extend Dry::Configurable
 
+    # WaterDrop options
     # option client_id [String] identifier of this producer
     setting :client_id, 'waterdrop'
-    # option logger [Instance, nil] logger that we want to use or nil to
-    #   fallback to ruby-kafka logger
-    setting :logger, nil
-    # Available options
-    setting :send_messages
-    # @option raise_on_failure [Boolean] Should raise error when failed to deliver a message
-    setting :raise_on_failure
+    # option [Instance, nil] logger that we want to use or nil to fallback to ruby-kafka logger
+    setting :logger, NullLogger.new
+    # option [Boolean] should we send messages. Setting this to false can be really useful when
+    #   testing and or developing because when set to false, won't actually ping Kafka
+    setting :deliver, true
 
-    # Connection pool options
-    setting :connection_pool do
-      # Connection pool size for producers. Note that we take a bigger number because there
-      # are cases when we might have more sidekiq threads than Karafka consumers (small app)
-      # or the opposite for bigger systems
-      setting :size, 2
-      # How long should we wait for a working resource from the pool before rising timeout
-      # With a proper connection pool size, this should never happen
-      setting :timeout, 5
-    end
-
-    # option kafka [Hash] - optional - kafka configuration options (hosts)
+    # Settings directly related to the Kafka driver
     setting :kafka do
-      # @option seed_brokers [Array<String>] Array that contains Kafka seed broker hosts with ports
+      # option [Array<String>] Array that contains Kafka seed broker hosts with ports
       setting :seed_brokers
+
+      # Network timeouts
       # option connect_timeout [Integer] Sets the number of seconds to wait while connecting to
       # a broker for the first time. When ruby-kafka initializes, it needs to connect to at
       # least one host.
@@ -38,15 +30,56 @@ module WaterDrop
       # writing to a socket connection to a broker. After this timeout expires the connection
       # will be killed. Note that some Kafka operations are by definition long-running, such as
       # waiting for new messages to arrive in a partition, so don't set this value too low
-      setting :socket_timeout, 10
+      setting :socket_timeout, 30
+
+      # Buffering for async producer
+      # @option [Integer] The maximum number of bytes allowed in the buffer before new messages
+      #   are rejected.
+      setting :max_buffer_bytesize, 10_000_000
+      # @option [Integer] The maximum number of messages allowed in the buffer before new messages
+      #   are rejected.
+      setting :max_buffer_size, 1000
+      # @option [Integer] The maximum number of messages allowed in the queue before new messages
+      #   are rejected. The queue is used to ferry messages from the foreground threads of your
+      #   application to the background thread that buffers and delivers messages.
+      setting :max_queue_size, 1000
+
+      # option [Integer] A timeout executed by a broker when the client is sending messages to it.
+      #   It defines the number of seconds the broker should wait for replicas to acknowledge the
+      #   write before responding to the client with an error. As such, it relates to the
+      #   required_acks setting. It should be set lower than socket_timeout.
+      setting :ack_timeout, 5
+      # option [Integer] The number of seconds between background message
+      #   deliveries. Default is 10 seconds. Disable timer-based background deliveries by
+      #   setting this to 0.
+      setting :delivery_interval, 10
+      # option [Integer] The number of buffered messages that will trigger a background message
+      #   delivery. Default is 100 messages. Disable buffer size based background deliveries by
+      #   setting this to 0.
+      setting :delivery_threshold, 100
+
+      # option [Integer] The number of retries when attempting to deliver messages.
+      setting :max_retries, 2
+      # option [Integer]
+      setting :required_acks, -1
+      # option [Integer]
+      setting :retry_backoff, 1
+
+      # option [Integer] The minimum number of messages that must be buffered before compression is
+      #   attempted. By default only one message is required. Only relevant if compression_codec
+      #   is set.
+      setting :compression_threshold, 1
+      # option [Symbol] The codec used to compress messages. Must be either snappy or gzip.
+      setting :compression_codec, nil
+
       # SSL authentication related settings
       # option ca_cert [String] SSL CA certificate
       setting :ssl_ca_cert, nil
       # option ssl_ca_cert_file_path [String] SSL CA certificate file path
       setting :ssl_ca_cert_file_path, nil
-      # option client_cert [String] SSL client certificate
+      # option ssl_client_cert [String] SSL client certificate
       setting :ssl_client_cert, nil
-      # option client_cert_key [String] SSL client certificate password
+      # option ssl_client_cert_key [String] SSL client certificate password
       setting :ssl_client_cert_key, nil
       # option sasl_gssapi_principal [String] sasl principal
       setting :sasl_gssapi_principal, nil
@@ -60,6 +93,17 @@ module WaterDrop
       setting :sasl_plain_password, nil
     end
 
+    # option producer [Hash] - optional - producer configuration options
+    setting :producer do
+      # option compression_codec [Symbol] Sets producer compression codec
+      # More: https://github.com/zendesk/ruby-kafka#compression
+      # More: https://github.com/zendesk/ruby-kafka/blob/master/lib/kafka/compression.rb
+      setting :compression_codec, nil
+      # option compression_codec [Integer] Sets producer compression threshold
+      # More: https://github.com/zendesk/ruby-kafka#compression
+      setting :compression_threshold, 1
+    end
+
     class << self
       # Configurating method
       # @yield Runs a block of code providing a config singleton instance to it
@@ -67,8 +111,21 @@ module WaterDrop
       def setup
         configure do |config|
           yield(config)
+          validate!(config.to_h)
         end
       end
+
+      private
+
+      # Validates the configuration and if anything is wrong, will raise an exception
+      # @param config_hash [Hash] config hash with setup details
+      # @raise [WaterDrop::Errors::InvalidConfiguration] raised when something is wrong with
+      #   the configuration
+      def validate!(config_hash)
+        validation_result = Schemas::Config.call(config_hash)
+        return true if validation_result.success?
+        raise Errors::InvalidConfiguration, validation_result.errors
+      end
     end
   end
 end

data/lib/water_drop/errors.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  # Namespace used to encapsulate all the internal errors of WaterDrop
+  module Errors
+    # Base class for all the WaterDrop internal errors
+    BaseError = Class.new(StandardError)
+
+    # Raised when configuration doesn't match with validation schema
+    InvalidConfiguration = Class.new(BaseError)
+
+    # Raised when we try to send message with invalid optionss
+    InvalidMessageOptions = Class.new(BaseError)
+  end
+end

data/lib/water_drop/schemas/config.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  # Namespace for all the schemas for config validations
+  module Schemas
+    # Schema with validation rules for WaterDrop configuration details
+    Config = Dry::Validation.Schema do
+      # Valid uri schemas of Kafka broker url
+      URI_SCHEMES = %w[
+        kafka
+        kafka+ssl
+      ].freeze
+
+      # All encryption related options keys
+      ENCRYPTION_OPTIONS_KEYS = %i[
+        ssl_ca_cert
+        ssl_ca_cert_file_path
+        ssl_client_cert
+        ssl_client_cert_key
+        sasl_plain_authzid
+        sasl_plain_username
+        sasl_plain_password
+        sasl_gssapi_principal
+        sasl_gssapi_keytab
+      ].freeze
+
+      configure do
+        config.messages_file = File.join(
+          WaterDrop.gem_root, 'config', 'errors.yml'
+        )
+
+        # Uri validator to check if uri is in a Kafka acceptable format
+        # @param uri [String] uri we want to validate
+        # @return [Boolean] true if it is a valid uri, otherwise false
+        def broker_schema?(uri)
+          uri = URI.parse(uri)
+          URI_SCHEMES.include?(uri.scheme) && uri.port
+        rescue URI::InvalidURIError
+          false
+        end
+      end
+
+      required(:client_id).filled(:str?, format?: Schemas::TOPIC_REGEXP)
+      required(:logger).filled
+      required(:deliver).filled(:bool?)
+
+      required(:kafka).schema do
+        required(:seed_brokers).filled { each(:broker_schema?) }
+        required(:connect_timeout).filled { (int? | float?) & gt?(0) }
+        required(:socket_timeout).filled { (int? | float?) & gt?(0) }
+        required(:compression_threshold).filled(:int?, gteq?: 1)
+        optional(:compression_codec).maybe(included_in?: %i[snappy gzip])
+
+        required(:max_buffer_bytesize).filled(:int?, gt?: 0)
+        required(:max_buffer_size).filled(:int?, gt?: 0)
+        required(:max_queue_size).filled(:int?, gt?: 0)
+
+        required(:ack_timeout).filled(:int?, gt?: 0)
+        required(:delivery_interval).filled(:int?, gteq?: 0)
+        required(:delivery_threshold).filled(:int?, gteq?: 0)
+
+        required(:max_retries).filled(:int?, gteq?: 0)
+        required(:retry_backoff).filled(:int?, gteq?: 0)
+        required(:required_acks).filled(included_in?: [1, 0, -1, :all])
+
+        ENCRYPTION_OPTIONS_KEYS.each do |encryption_attribute|
+          optional(encryption_attribute).maybe(:str?)
+        end
+      end
+    end
+  end
+end

data/lib/water_drop/schemas/message_options.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Schemas
+    # Regexp to check that topic has a valid format
+    TOPIC_REGEXP = /\A(\w|\-|\.)+\z/
+
+    # Schema with validation rules for validating that all the message options that
+    # we provide to producer ale valid and usable
+    # @note Does not validate message itself as it is not our concern
+    MessageOptions = Dry::Validation.Schema do
+      required(:topic).filled(:str?, format?: TOPIC_REGEXP)
+      optional(:key).maybe(:str?, :filled?)
+      optional(:partition).filled(:int?, gteq?: 0)
+      optional(:partition_key).maybe(:str?, :filled?)
+    end
+  end
+end

data/lib/water_drop/sync_producer.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+# WaterDrop library
+module WaterDrop
+  # Sync producer for messages
+  SyncProducer = Class.new(BaseProducer)
+  # Sync producer for messages
+  Producer = SyncProducer
+  SyncProducer.method_name = :deliver
+end

data/lib/water_drop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '0.4.0'
+  VERSION = '1.0.0.alpha1'
 end

data/waterdrop.gemspec

@@ -9,22 +9,18 @@ Gem::Specification.new do |spec|
   spec.name          = 'waterdrop'
   spec.version       = ::WaterDrop::VERSION
   spec.platform      = Gem::Platform::RUBY
-  spec.authors       = ['Maciej Mensfeld', 'Pavlo Vavruk']
-  spec.email         = %w[maciej@mensfeld.pl pavlo.vavruk@gmail.com]
+  spec.authors       = ['Maciej Mensfeld']
+  spec.email         = %w[maciej@mensfeld.pl]
   spec.homepage      = 'https://github.com/karafka/waterdrop'
   spec.summary       = ' Kafka messaging made easy! '
   spec.description   = spec.summary
   spec.license       = 'MIT'
 
-  spec.add_dependency 'bundler', '>= 0'
-  spec.add_dependency 'rake', '>= 0'
-  spec.add_dependency 'ruby-kafka', '~> 0.4'
-  spec.add_dependency 'connection_pool', '>= 0'
+  spec.add_dependency 'delivery_boy', '>= 0.2.2'
+  spec.add_dependency 'dry-configurable', '~> 0.7'
+  spec.add_dependency 'dry-validation', '~> 0.11'
   spec.add_dependency 'null-logger'
-  spec.add_dependency 'dry-configurable', '~> 0.6'
 
-  spec.add_development_dependency 'rspec', '~> 3.6.0'
-  spec.add_development_dependency 'simplecov', '~> 0.14.1'
   spec.required_ruby_version = '>= 2.2.0'
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }

metadata

@@ -1,72 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 1.0.0.alpha1
 platform: ruby
 authors:
 - Maciej Mensfeld
-- Pavlo Vavruk
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-11 00:00:00.000000000 Z
+date: 2017-10-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: delivery_boy
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.2.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.2.2
 - !ruby/object:Gem::Dependency
-  name: ruby-kafka
+  name: dry-configurable
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: '0.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: '0.7'
 - !ruby/object:Gem::Dependency
-  name: connection_pool
+  name: dry-validation
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.11'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.11'
 - !ruby/object:Gem::Dependency
   name: null-logger
   requirement: !ruby/object:Gem::Requirement
@@ -81,52 +66,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: dry-configurable
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.6'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.6'
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 3.6.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 3.6.0
-- !ruby/object:Gem::Dependency
-  name: simplecov
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.14.1
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.14.1
 description: Kafka messaging made easy!
 email:
 - maciej@mensfeld.pl
-- pavlo.vavruk@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -141,12 +83,15 @@ files:
 - Gemfile.lock
 - MIT-LICENCE
 - README.md
-- Rakefile
+- config/errors.yml
 - lib/water_drop.rb
+- lib/water_drop/async_producer.rb
+- lib/water_drop/base_producer.rb
 - lib/water_drop/config.rb
-- lib/water_drop/message.rb
-- lib/water_drop/pool.rb
-- lib/water_drop/producer_proxy.rb
+- lib/water_drop/errors.rb
+- lib/water_drop/schemas/config.rb
+- lib/water_drop/schemas/message_options.rb
+- lib/water_drop/sync_producer.rb
 - lib/water_drop/version.rb
 - lib/waterdrop.rb
 - waterdrop.gemspec
@@ -165,12 +110,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.2.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: Kafka messaging made easy!

data/Rakefile

@@ -1,8 +0,0 @@
-# frozen_string_literal: true
-
-require 'bundler'
-require 'rake'
-require 'rspec/core/rake_task'
-
-RSpec::Core::RakeTask.new(:spec)
-task default: :spec

data/lib/water_drop/message.rb

@@ -1,40 +0,0 @@
-# frozen_string_literal: true
-
-module WaterDrop
-  # Message class which encapsulate single Kafka message logic and its delivery
-  class Message
-    attr_reader :topic, :message, :options
-
-    # @param topic [String, Symbol] a topic to which we want to send a message
-    # @param message [Object] any object that can be serialized to a JSON string or
-    #   that can be casted to a string
-    # @param options [Hash] (optional) additional options to pass to the Kafka producer
-    # @return [WaterDrop::Message] WaterDrop message instance
-    # @example Creating a new message
-    #   WaterDrop::Message.new(topic, message)
-    def initialize(topic, message, options = {})
-      @topic = topic.to_s
-      @message = message
-      @options = options
-    end
-
-    # Sents a current message to Kafka
-    # @note Won't send any messages if send_messages config flag is set to false
-    # @example Set a message
-    #   WaterDrop::Message.new(topic, message).send!
-    def send!
-      return true unless ::WaterDrop.config.send_messages
-
-      Pool.with { |producer| producer.send_message(self) }
-
-      ::WaterDrop.logger.info("Message #{message} was sent to topic '#{topic}'")
-    rescue StandardError => e
-      # Even if we dont reraise this exception, it should log that it happened
-      ::WaterDrop.logger.error(e)
-      # Reraise if we want to raise on failure
-      # Ignore if we dont want to know that something went wrong
-      return unless ::WaterDrop.config.raise_on_failure
-      raise(e)
-    end
-  end
-end

data/lib/water_drop/pool.rb

@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-module WaterDrop
-  # Raw Kafka producers connection pool for WaterDrop messages delivery
-  module Pool
-    extend SingleForwardable
-    # Delegate directly to pool
-    def_delegators :pool, :with
-
-    class << self
-      # @return [::ConnectionPool] connection pool instance that we can then use
-      def pool
-        @pool ||= ConnectionPool.new(
-          size: ::WaterDrop.config.connection_pool.size,
-          timeout: ::WaterDrop.config.connection_pool.timeout
-        ) do
-          WaterDrop::ProducerProxy.new
-        end
-      end
-    end
-  end
-end

data/lib/water_drop/producer_proxy.rb

@@ -1,80 +0,0 @@
-# frozen_string_literal: true
-
-module WaterDrop
-  # Proxy object for a producer (sender) objects that are inside pool
-  # We use it to provide additional timeout monitoring layer
-  # There used to be an issue with Poseidon (previous engine for this lib)
-  # usage of sockets that are old and not used - that's why we just
-  # reinitialize connection if the connection layer is not being used for too long
-  # We keep this logic to avoid problems just in case. If those problems won't occur
-  # with Ruby-Kafka, we will drop it
-  class ProducerProxy
-    # How long should be object considered alive if nothing is being
-    # send using it. After that time, we will recreate the connection
-    LIFE_TIME = 5 * 60 #  5 minute
-
-    # If sending fails - how many times we should try with a new connection
-    MAX_SEND_RETRIES = 1
-
-    # @return [WaterDrop::ProducerProxy] proxy object to Kafka::Producer
-    # @note To ignore @last_usage nil case - we just assume that it is being
-    #   first used when we create it
-    def initialize
-      touch
-      @attempts = 0
-    end
-
-    # Sends message to Kafka
-    # @param message [WaterDrop::Message] message that we want to send
-    # @note If something goes wrong it will assume that producer is corrupted and will try to
-    #   create a new one
-    # @example Send 1 message
-    #   ProducerProxy.new.send_message(WaterDrop::Message.new(topic, message))
-    def send_message(message)
-      touch
-      producer.produce(
-        message.message,
-        { topic: message.topic }.merge(message.options)
-      )
-      producer.deliver_messages
-    rescue StandardError => e
-      reload!
-
-      retry if (@attempts += 1) <= MAX_SEND_RETRIES
-
-      raise(e)
-    ensure
-      @attempts = 0
-    end
-
-    private
-
-    # Refreshes last usage value with current time
-    def touch
-      @last_usage = Time.now
-    end
-
-    # @return [Kafka::Producer] producer instance to which we can forward method requests
-    def producer
-      reload! if dead?
-      @producer ||= Kafka.new(
-        {
-          client_id: ::WaterDrop.config.client_id,
-          logger: ::WaterDrop.config.logger
-        }.merge(::WaterDrop.config.kafka.to_h)
-      ).producer
-    end
-
-    # @return [Boolean] true if we cannot use producer anymore because it was not used for a
-    #   long time
-    def dead?
-      @last_usage + LIFE_TIME < Time.now
-    end
-
-    # Resets a producer so a new one will be created once requested
-    def reload!
-      @producer&.shutdown
-      @producer = nil
-    end
-  end
-end