waterdrop 0.3.2.4 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b8146a30bc7d849c7e9bd8d5ab6a3e65697467a4
-  data.tar.gz: b6b606e0194c98093293fd42fb891d821a2937a7
+  metadata.gz: d6ebd8fb5186e625ad83a4039b1953e145e6ed6e
+  data.tar.gz: 3f5b6e5d6ac383388a4fff4d18f6600122e91bbc
 SHA512:
-  metadata.gz: 5e49033efc2f820a6c61591f475f9ff2c9782b8867a10b95ff31d4a375f0979913aa96315ac93ade25befbad558309ebc95c1015de98b21933c19b7b745a8c7c
-  data.tar.gz: 54a112efc2652be8e0315b183f7fb3e7c16322d3cd67ef145d3f49c6ba172baf9cffdd374b2651bfe07c9500b150cb5d78d6c6ae4da73e31368121dd58d8faa6
+  metadata.gz: 2ffc2c7b34e2e9f53b61db43de61fe5f557af524ffba98f1101a1a630d5af021d90d76e0d65a0c06a2d13114933240d9f984d0b359251ac58af84e2afff04320
+  data.tar.gz: d1617c0169e356b16332d9b70285e58f0b524185ca2cc413903d53316f379bf1a58a1d383c990883cb2eb827194a6c9d2959d3459a3f04f99548c3c2d5394f47

data/.rspec

@@ -1,4 +1 @@
---color
---format documentation
 --require spec_helper
---tty

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # WaterDrop changelog
 
+## 0.4 unreleased
+- Bump to match Karafka
+- Renamed ```hosts``` to ```seed_brokers```
+- Removed the ```ssl``` scoping for ```kafka``` config namespace to better match Karafka conventions
+- Added ```client_id``` option on a root config level
+- Added ```logger``` option on a root config level
+- Auto Propagation of config down to ruby-kafka
+
 ## 0.3.2
 - Removed support for Ruby 2.1.*
 - ~~Ruby 2.3.3 as default~~

data/Gemfile.lock

@@ -1,13 +1,13 @@
 PATH
   remote: .
   specs:
-    waterdrop (0.3.2.4)
+    waterdrop (0.4.0)
       bundler
       connection_pool
       dry-configurable (~> 0.6)
       null-logger
       rake
-      ruby-kafka
+      ruby-kafka (~> 0.4)
 
 GEM
   remote: https://rubygems.org/
@@ -18,6 +18,9 @@ GEM
     docile (1.1.5)
     dry-configurable (0.7.0)
       concurrent-ruby (~> 1.0)
+    ffi (1.9.18)
+    gssapi (1.2.0)
+      ffi (>= 1.0.1)
     json (2.1.0)
     null-logger (0.1.4)
     rake (12.0.0)
@@ -34,7 +37,8 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.6.0)
     rspec-support (3.6.0)
-    ruby-kafka (0.3.17)
+    ruby-kafka (0.4.0)
+      gssapi (>= 1.2.0)
     simplecov (0.14.1)
       docile (~> 1.1.0)
       json (>= 1.8, < 3)

data/README.md

@@ -29,25 +29,33 @@ bundle install
 
 WaterDrop has following configuration options:
 
-| Option                    | Required   | Value type    | Description                      |
-|---------------------------|------------|---------------|----------------------------------|
-| send_messages             | true       | Boolean       | Should we send messages to Kafka |
-| kafka.hosts               | true       | Array<String> | Kafka servers hosts with ports   |
-| connection_pool_size      | true       | Integer       | Kafka connection pool size       |
-| connection_pool_timeout   | true       | Integer       | Kafka connection pool timeout    |
-| raise_on_failure          | true       | Boolean       | Should we raise an exception when we cannot send message to Kafka - if false will silently ignore failures (will just ignore them) |
-| kafka.ssl.ca_cert         | false      | String        | SSL CA certificate               |
-| kafka.ssl.client_cert     | false      | String        | SSL client certificate           |
-| kafka.ssl.client_cert_key | false      | String        | SSL client certificate password  |
+| Option                      | Required   | Value type    | Description                                                                        |
+|-----------------------------|------------|---------------|------------------------------------------------------------------------------------|
+| send_messages               | true       | Boolean       | Should we send messages to Kafka                                                   |
+| connect_timeout             | false      | Integer       | Number of seconds to wait while connecting to a broker for the first time          |
+| socket_timeout              | false      | Integer       | Number of seconds to wait when reading from or writing to a socket                 |
+| connection_pool.size        | true       | Integer       | Kafka connection pool size                                                         |
+| connection_pool.timeout     | true       | Integer       | Kafka connection pool timeout                                                      |
+| kafka.seed_brokers          | true       | Array<String> | Kafka servers hosts with ports                                                     |
+| raise_on_failure            | true       | Boolean       | Should we raise an exception when we cannot send message to Kafka - if false will silently ignore failures |
+| kafka.ssl_ca_cert           | false      | String        | SSL CA certificate                                                                 |
+| kafka.ssl_ca_cert_file_path | false      | String        | SSL CA certificate file path                                                       |
+| kafka.ssl_client_cert       | false      | String        | SSL client certificate                                                             |
+| kafka.ssl_client_cert_key   | false      | String        | SSL client certificate password                                                    |
+| kafka.sasl_gssapi_principal | false      | String        | SASL principal                                                                     |
+| kafka.sasl_gssapi_keytab    | false      | String        | SASL keytab                                                                        |
+| kafka.sasl_plain_authzid    | false      | String        | The authorization identity to use                                                  |
+| kafka.sasl_plain_username   | false      | String        | The username used to authenticate                                                  |
+| kafka.sasl_plain_password   | false      | String        | The password used to authenticate                                                  |
 
 To apply this configuration, you need to use a *setup* method:
 
 ```ruby
 WaterDrop.setup do |config|
   config.send_messages = true
-  config.connection_pool_size = 20
-  config.connection_pool_timeout = 1
-  config.kafka.hosts = ['localhost:9092']
+  config.connection_pool.size = 20
+  config.connection_pool.timeout = 1
+  config.kafka.seed_brokers = ['localhost:9092']
   config.raise_on_failure = true
 end
 ```
@@ -57,9 +65,9 @@ This configuration can be placed in *config/initializers* and can vary based on
 ```ruby
 WaterDrop.setup do |config|
   config.send_messages = Rails.env.production?
-  config.connection_pool_size = 20
-  config.connection_pool_timeout = 1
-  config.kafka.hosts = [Rails.env.production? ? 'prod-host:9091' : 'localhost:9092']
+  config.connection_pool.size = 20
+  config.connection_pool.timeout = 1
+  config.kafka.seed_brokers = [Rails.env.production? ? 'prod-host:9091' : 'localhost:9092']
   config.raise_on_failure = Rails.env.production?
 end
 ```

data/Rakefile

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'bundler'
 require 'rake'
 require 'rspec/core/rake_task'

data/lib/water_drop/config.rb

@@ -5,29 +5,59 @@ module WaterDrop
   class Config
     extend Dry::Configurable
 
+    # option client_id [String] identifier of this producer
+    setting :client_id, 'waterdrop'
+    # option logger [Instance, nil] logger that we want to use or nil to
+    #   fallback to ruby-kafka logger
+    setting :logger, nil
     # Available options
-    # @option connection_pool_timeout [Fixnum] Amount of time in seconds to wait for a connection
-    #         if none currently available.
-    setting :connection_pool_timeout
-    # @option send_messages [Boolean] boolean value to define whether messages should be sent
     setting :send_messages
     # @option raise_on_failure [Boolean] Should raise error when failed to deliver a message
     setting :raise_on_failure
-    # @option connection_pool_size [Fixnum] The number of connections to pool.
-    setting :connection_pool_size
+
+    # Connection pool options
+    setting :connection_pool do
+      # Connection pool size for producers. Note that we take a bigger number because there
+      # are cases when we might have more sidekiq threads than Karafka consumers (small app)
+      # or the opposite for bigger systems
+      setting :size, 2
+      # How long should we wait for a working resource from the pool before rising timeout
+      # With a proper connection pool size, this should never happen
+      setting :timeout, 5
+    end
+
     # option kafka [Hash] - optional - kafka configuration options (hosts)
     setting :kafka do
-      # @option hosts [Array<String>] Array that contains Kafka hosts with ports
-      setting :hosts
+      # @option seed_brokers [Array<String>] Array that contains Kafka seed broker hosts with ports
+      setting :seed_brokers
+      # option connect_timeout [Integer] Sets the number of seconds to wait while connecting to
+      # a broker for the first time. When ruby-kafka initializes, it needs to connect to at
+      # least one host.
+      setting :connect_timeout, 10
+      # option socket_timeout [Integer] Sets the number of seconds to wait when reading from or
+      # writing to a socket connection to a broker. After this timeout expires the connection
+      # will be killed. Note that some Kafka operations are by definition long-running, such as
+      # waiting for new messages to arrive in a partition, so don't set this value too low
+      setting :socket_timeout, 10
       # SSL authentication related settings
-      setting :ssl do
-        # option ca_cert [String] SSL CA certificate
-        setting :ca_cert, nil
-        # option client_cert [String] SSL client certificate
-        setting :client_cert, nil
-        # option client_cert_key [String] SSL client certificate password
-        setting :client_cert_key, nil
-      end
+      # option ca_cert [String] SSL CA certificate
+      setting :ssl_ca_cert, nil
+      # option ssl_ca_cert_file_path [String] SSL CA certificate file path
+      setting :ssl_ca_cert_file_path, nil
+      # option client_cert [String] SSL client certificate
+      setting :ssl_client_cert, nil
+      # option client_cert_key [String] SSL client certificate password
+      setting :ssl_client_cert_key, nil
+      # option sasl_gssapi_principal [String] sasl principal
+      setting :sasl_gssapi_principal, nil
+      # option sasl_gssapi_keytab [String] sasl keytab
+      setting :sasl_gssapi_keytab, nil
+      # option sasl_plain_authzid [String] The authorization identity to use
+      setting :sasl_plain_authzid, ''
+      # option sasl_plain_username [String] The username used to authenticate
+      setting :sasl_plain_username, nil
+      # option sasl_plain_password [String] The password used to authenticate
+      setting :sasl_plain_password, nil
     end
 
     class << self

data/lib/water_drop/pool.rb

@@ -11,8 +11,8 @@ module WaterDrop
       # @return [::ConnectionPool] connection pool instance that we can then use
       def pool
         @pool ||= ConnectionPool.new(
-          size: ::WaterDrop.config.connection_pool_size,
-          timeout: ::WaterDrop.config.connection_pool_timeout
+          size: ::WaterDrop.config.connection_pool.size,
+          timeout: ::WaterDrop.config.connection_pool.timeout
         ) do
           WaterDrop::ProducerProxy.new
         end

data/lib/water_drop/producer_proxy.rb

@@ -32,9 +32,10 @@ module WaterDrop
     #   ProducerProxy.new.send_message(WaterDrop::Message.new(topic, message))
     def send_message(message)
       touch
-      producer.produce(message.message, {
-        topic: message.topic
-      }.merge(message.options))
+      producer.produce(
+        message.message,
+        { topic: message.topic }.merge(message.options)
+      )
       producer.deliver_messages
     rescue StandardError => e
       reload!
@@ -57,10 +58,10 @@ module WaterDrop
     def producer
       reload! if dead?
       @producer ||= Kafka.new(
-        seed_brokers: ::WaterDrop.config.kafka.hosts,
-        ssl_ca_cert: ::WaterDrop.config.kafka.ssl.ca_cert,
-        ssl_client_cert: ::WaterDrop.config.kafka.ssl.client_cert,
-        ssl_client_cert_key: ::WaterDrop.config.kafka.ssl.client_cert_key
+        {
+          client_id: ::WaterDrop.config.client_id,
+          logger: ::WaterDrop.config.logger
+        }.merge(::WaterDrop.config.kafka.to_h)
       ).producer
     end
 

data/lib/water_drop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '0.3.2.4'
+  VERSION = '0.4.0'
 end

data/waterdrop.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'bundler', '>= 0'
   spec.add_dependency 'rake', '>= 0'
-  spec.add_dependency 'ruby-kafka', '>= 0'
+  spec.add_dependency 'ruby-kafka', '~> 0.4'
   spec.add_dependency 'connection_pool', '>= 0'
   spec.add_dependency 'null-logger'
   spec.add_dependency 'dry-configurable', '~> 0.6'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 0.3.2.4
+  version: 0.4.0
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-30 00:00:00.000000000 Z
+date: 2017-08-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -43,16 +43,16 @@ dependencies:
   name: ruby-kafka
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.4'
 - !ruby/object:Gem::Dependency
   name: connection_pool
   requirement: !ruby/object:Gem::Requirement