waterdrop 1.0.1 → 1.2.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 4c194065166002b613a559ba2818edc33edea127
-  data.tar.gz: ca654d85779ccefd488269360a977db8e3406f54
+SHA256:
+  metadata.gz: 25c083d88a1163c685e9939307feba4294fd3a526f4523692958cdc5d0d04363
+  data.tar.gz: 300d3e1cce18970d0c9beb105ab6e3b6b7d5eedfc388e3fad7f1cd0db1c204e3
 SHA512:
-  metadata.gz: dee9f25037979516a0f5d62591e4cce4706a2f54b10862e63bc0ea757664f6dc7b442efb501f70e9eafe0b1ec9d6389be0f9b74968369faeefa69ad1adf864d4
-  data.tar.gz: da09ca5c22c4553e0c6f82fb16a7761bf97453c8f6ac8a3713ac350f09ffe69f47150912a48f4c2574b9bdacc6197d9ad445dff2ee4c030a082035e5c8ef33b1
+  metadata.gz: aa9f0cdfb2285b1f50a71f9a1e84310903e1abc936cd41838f8568a2ddf818b4c83324ccaf7a7c299980a7a8b7d6f0c8fb7fdad65b075e26440bc740be1f20cb
+  data.tar.gz: fff2c225e42eeb1ab849fb5ea34c900e6bf4b9783541a6087c094647ec8235c6ce6305087119455d2900cfd53c55e7179b27be5b0f29d49ef80f70720f6674ff

data/.ruby-version

@@ -1 +1 @@
-2.4.2
+2.5.0

data/.travis.yml

@@ -9,6 +9,7 @@ rvm:
   - 2.4.0
   - 2.4.1
   - 2.4.2
+  - 2.5.0
   - jruby-head
 script: bundle exec rspec spec/
 env:

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 # WaterDrop changelog
 
+## 1.2.0.beta1
+- #45 - Allow specifying a create time for messages
+- #47 - Support SCRAM once released
+- #49 - Add lz4 support once merged and released
+- #50 - Potential message loss in async mode
+- Ruby 2.5.0 support
+- Gem bump to match Karafka framework versioning
+- #48 - ssl_ca_certs_from_system
+- #52 - Use instrumentation compatible with Karafka 1.2
+
 ## 1.0.1
 - Added high level retry on connection problems
 

data/Gemfile.lock

@@ -1,20 +1,21 @@
 PATH
   remote: .
   specs:
-    waterdrop (1.0.1)
-      delivery_boy (>= 0.2.2)
+    waterdrop (1.2.0.beta1)
+      delivery_boy (>= 0.2.3)
       dry-configurable (~> 0.7)
+      dry-monitor (~> 0.1)
       dry-validation (~> 0.11)
       null-logger
-      ruby-kafka (>= 0.5)
+      ruby-kafka (>= 0.5.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
     concurrent-ruby (1.0.5)
-    delivery_boy (0.2.2)
+    delivery_boy (0.2.4)
       king_konf (~> 0.1.8)
-      ruby-kafka (~> 0.4)
+      ruby-kafka (~> 0.5.1)
     diff-lcs (1.3)
     docile (1.1.5)
     dry-configurable (0.7.0)
@@ -22,13 +23,22 @@ GEM
     dry-container (0.6.0)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.1, >= 0.1.3)
-    dry-core (0.4.1)
+    dry-core (0.4.4)
       concurrent-ruby (~> 1.0)
     dry-equalizer (0.2.0)
+    dry-events (0.1.0)
+      concurrent-ruby (~> 1.0)
+      dry-core (~> 0.4)
+      dry-equalizer (~> 0.2)
     dry-logic (0.4.2)
       dry-container (~> 0.2, >= 0.2.6)
       dry-core (~> 0.2)
       dry-equalizer (~> 0.2)
+    dry-monitor (0.1.2)
+      dry-configurable (~> 0.5)
+      dry-equalizer (~> 0.2)
+      dry-events (~> 0.1)
+      rouge (~> 2.0, >= 2.2.1)
     dry-types (0.12.2)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.1)
@@ -46,13 +56,14 @@ GEM
       dry-types (~> 0.12.0)
     inflecto (0.0.2)
     json (2.1.0)
-    king_konf (0.1.8)
+    king_konf (0.1.10)
     null-logger (0.1.4)
+    rouge (2.2.1)
     rspec (3.7.0)
       rspec-core (~> 3.7.0)
       rspec-expectations (~> 3.7.0)
       rspec-mocks (~> 3.7.0)
-    rspec-core (3.7.0)
+    rspec-core (3.7.1)
       rspec-support (~> 3.7.0)
     rspec-expectations (3.7.0)
       diff-lcs (>= 1.2.0, < 2.0)
@@ -60,8 +71,8 @@ GEM
     rspec-mocks (3.7.0)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.7.0)
-    rspec-support (3.7.0)
-    ruby-kafka (0.5.0)
+    rspec-support (3.7.1)
+    ruby-kafka (0.5.3)
     simplecov (0.15.1)
       docile (~> 1.1.0)
       json (>= 1.8, < 3)
@@ -77,4 +88,4 @@ DEPENDENCIES
   waterdrop!
 
 BUNDLED WITH
-   1.15.4
+   1.16.1

data/README.md

@@ -11,6 +11,7 @@ It is:
 
  - Thread safe
  - Supports sync and async producers
+ - Working with 0.10.1+ Kafka
 
 ## Installation
 
@@ -60,19 +61,20 @@ end
 
 **Note:** All the options are subject to validations. In order to check what is and what is not acceptable, please go to the [config.rb validation schema](https://github.com/karafka/waterdrop/blob/master/lib/water_drop/schemas/config.rb) file.
 
-| Option              | Description                                                                                                                                           |
-|---------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------|
-| delivery_interval   | The number of seconds between background message deliveries. Disable timer-based background deliveries by setting this to 0.                          |
-| delivery_threshold  | The number of buffered messages that will trigger a background message delivery. Disable buffer size based background deliveries by setting this to 0.|
-| required_acks       | The number of Kafka replicas that must acknowledge messages before they're considered as successfully written.                                        |
-| ack_timeout         | A timeout executed by a broker when the client is sending messages to it.                                                                             |
-| max_retries         | The number of retries when attempting to deliver messages.                                                                                            |
-| retry_backoff       | The number of seconds to wait after a failed attempt to send messages to a Kafka broker before retrying.                                              |
-| max_buffer_bytesize | The maximum number of bytes allowed in the buffer before new messages are rejected.                                                                   |
-| max_buffer_size     | The maximum number of messages allowed in the buffer before new messages are rejected.                                                                |
-| max_queue_size      | The maximum number of messages allowed in the queue before new messages are rejected.                                                                 |
-| sasl_plain_username | The username used to authenticate.                                                                                                                    |
-| sasl_plain_password | The password used to authenticate.                                                                                                                    |
+| Option                   | Description                                                                                                                                           |
+|--------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------|
+| raise_on_buffer_overflow | Should we raise an exception, when messages can't be sent in an async way due to the message buffer overflow or should we just drop them              |
+| delivery_interval        | The number of seconds between background message deliveries. Disable timer-based background deliveries by setting this to 0.                          |
+| delivery_threshold       | The number of buffered messages that will trigger a background message delivery. Disable buffer size based background deliveries by setting this to 0.|
+| required_acks            | The number of Kafka replicas that must acknowledge messages before they're considered as successfully written.                                        |
+| ack_timeout              | A timeout executed by a broker when the client is sending messages to it.                                                                             |
+| max_retries              | The number of retries when attempting to deliver messages.                                                                                            |
+| retry_backoff            | The number of seconds to wait after a failed attempt to send messages to a Kafka broker before retrying.                                              |
+| max_buffer_bytesize      | The maximum number of bytes allowed in the buffer before new messages are rejected.                                                                   |
+| max_buffer_size          | The maximum number of messages allowed in the buffer before new messages are rejected.                                                                |
+| max_queue_size           | The maximum number of messages allowed in the queue before new messages are rejected.                                                                 |
+| sasl_plain_username      | The username used to authenticate.                                                                                                                    |
+| sasl_plain_password      | The password used to authenticate.                                                                                                                    |
 
 This configuration can be also placed in *config/initializers* and can vary based on the environment:
 
@@ -95,12 +97,13 @@ WaterDrop::AsyncProducer.call('message', topic: 'my-topic')
 
 Both ```SyncProducer``` and ```AsyncProducer``` accept following options:
 
-| Option              | Required | Value type     | Description                                                         |
-|-------------------- |----------|----------------|---------------------------------------------------------------------|
-| ```topic```         | true     | String         | The Kafka topic that should be written to                           |
-| ```key```           | false    | String         | The key that should be set on the Kafka message                     |
-| ```partition```     | false    | Integer        | A specific partition number that should be written to               |
-| ```partition_key``` | false    | String         | A string that can be used to deterministically select the partition |
+| Option              | Required | Value type | Description                                                         |
+|-------------------- |----------|------------|---------------------------------------------------------------------|
+| ```topic```         | true     | String     | The Kafka topic that should be written to                           |
+| ```key```           | false    | String     | The key that should be set on the Kafka message                     |
+| ```partition```     | false    | Integer    | A specific partition number that should be written to               |
+| ```partition_key``` | false    | String     | A string that can be used to deterministically select the partition |
+| ```create_time```   | false    | Time       | The timestamp that should be set on the message                     |
 
 Keep in mind, that message you want to send should be either binary or stringified (to_s, to_json, etc).
 
@@ -110,23 +113,14 @@ Keep in mind, that message you want to send should be either binary or stringifi
 * [WaterDrop Travis CI](https://travis-ci.org/karafka/waterdrop)
 * [WaterDrop Coditsu](https://app.coditsu.io/karafka/repositories/waterdrop)
 
-## Note on Patches/Pull Requests
+## Note on contributions
 
-Fork the project.
-Make your feature addition or bug fix.
-Add tests for it. This is important so we don't break it in a future versions unintentionally.
-Commit, do not mess with version, or history. (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull). Send me a pull request. Bonus points for topic branches.
+First, thank you for considering contributing to WaterDrop! It's people like you that make the open source community such a great community!
 
-[![coditsu](https://coditsu.io/assets/quality_bar.svg)](https://app.coditsu.io/karafka/repositories/waterdrop)
+Each pull request must pass all the rspec specs and meet our quality requirements.
 
-Each pull request must pass our quality requirements. To check if everything is as it should be, we use [Coditsu](https://coditsu.io) that combines multiple linters and code analyzers for both code and documentation.
+To check if everything is as it should be, we use [Coditsu](https://coditsu.io) that combines multiple linters and code analyzers for both code and documentation. Once you're done with your changes, submit a pull request.
 
-Unfortunately, it does not yet support independent forks, however you should be fine by looking at what we require.
+Coditsu will automatically check your work against our quality standards. You can find your commit check results on the [builds page](https://app.coditsu.io/karafka/repositories/waterdrop/builds/commit_builds) of WaterDrop repository.
 
-Please run:
-
-```bash
-bundle exec rspec
-```
-
-to check if everything is in order. After that you can submit a pull request.
+[![coditsu](https://coditsu.io/assets/quality_bar.svg)](https://app.coditsu.io/karafka/repositories/waterdrop/builds/commit_builds)

data/lib/water_drop.rb

@@ -5,7 +5,9 @@
   json
   delivery_boy
   null_logger
+  singleton
   dry-configurable
+  dry/monitor/notifications
   dry-validation
 ].each { |lib| require lib }
 
@@ -44,6 +46,11 @@ module WaterDrop
       Config.config
     end
 
+    # @return [::WaterDrop::Monitor] monitor that we want to use
+    def monitor
+      config.monitor
+    end
+
     # @return [String] root path of this gem
     def gem_root
       Pathname.new(File.expand_path('../..', __FILE__))
@@ -53,6 +60,8 @@ end
 
 %w[
   version
+  instrumentation/monitor
+  instrumentation/listener
   schemas/message_options
   schemas/config
   config

data/lib/water_drop/async_producer.rb

@@ -10,20 +10,17 @@ module WaterDrop
     # @raise [WaterDrop::Errors::InvalidMessageOptions] raised when message options are
     #   somehow invalid and we cannot perform delivery because of that
     def self.call(message, options)
-      attempts ||= 0
-      attempts += 1
+      attempts_count ||= 0
+      attempts_count += 1
 
       validate!(options)
       return unless WaterDrop.config.deliver
-      DeliveryBoy.deliver_async(message, options)
-    rescue Kafka::Error => e
-      if attempts > WaterDrop.config.kafka.max_retries
-        WaterDrop.logger.error e
-        raise e
-      else
-        WaterDrop.logger.warn "Retrying delivery after: #{e}"
-        retry
-      end
+
+      d_method = WaterDrop.config.raise_on_buffer_overflow ? :deliver_async! : :deliver_async
+
+      DeliveryBoy.send(d_method, message, options)
+    rescue Kafka::Error => error
+      graceful_attempt?(attempts_count, message, options, error) ? retry : raise(error)
     end
   end
 end

data/lib/water_drop/base_producer.rb

@@ -16,6 +16,35 @@ module WaterDrop
         return true if validation_result.success?
         raise Errors::InvalidMessageOptions, validation_result.errors
       end
+
+      # Upon failed delivery, we may try to resend a message depending on the attempt number
+      #   or reraise an error if we're unable to do that after given number of retries
+      #   This method checks that and also instruments errors and retries for the delivery
+      # @param attempts_count [Integer] number of attempt (starting from 1) for the delivery
+      # @param message [String] message that we want to send to Kafka
+      # @param options [Hash] options (including topic) for producer
+      # @param error [Kafka::Error] error that occured
+      # @return [Boolean] true if this is a graceful attempt and we can retry or false it this
+      #   was the final one and we should deal with the fact, that we cannot deliver a given
+      #   message
+      def graceful_attempt?(attempts_count, message, options, error)
+        scope = "#{to_s.split('::').last.sub('Producer', '_producer').downcase}.call"
+        payload = {
+          caller: self,
+          message: message,
+          options: options,
+          error: error,
+          attempts_count: attempts_count
+        }
+
+        if attempts_count > WaterDrop.config.kafka.max_retries
+          WaterDrop.monitor.instrument("#{scope}.error", payload)
+          false
+        else
+          WaterDrop.monitor.instrument("#{scope}.retry", payload)
+          true
+        end
+      end
     end
   end
 end

data/lib/water_drop/config.rb

@@ -12,9 +12,17 @@ module WaterDrop
     setting :client_id, 'waterdrop'
     # option [Instance, nil] logger that we want to use or nil to fallback to ruby-kafka logger
     setting :logger, NullLogger.new
+    # option [Instance] monitor that we want to use. See instrumentation part of the README for
+    #   more details
+    setting :monitor, WaterDrop::Instrumentation::Monitor.instance
     # option [Boolean] should we send messages. Setting this to false can be really useful when
     #   testing and or developing because when set to false, won't actually ping Kafka
     setting :deliver, true
+    # option [Boolean] if you're producing messages faster than the framework or the network can
+    #   send them off, ruby-kafka might reject them. If that happens, WaterDrop will either raise
+    #   or ignore - this setting manages that behavior. This only applies to async producer as
+    #   sync producer will always raise upon problems
+    setting :raise_on_buffer_overflow, true
 
     # Settings directly related to the Kafka driver
     setting :kafka do
@@ -73,24 +81,33 @@ module WaterDrop
       setting :compression_codec, nil
 
       # SSL authentication related settings
-      # option ca_cert [String] SSL CA certificate
+      # option ca_cert [String, nil] SSL CA certificate
       setting :ssl_ca_cert, nil
-      # option ssl_ca_cert_file_path [String] SSL CA certificate file path
+      # option ssl_ca_cert_file_path [String, nil] SSL CA certificate file path
       setting :ssl_ca_cert_file_path, nil
-      # option ssl_client_cert [String] SSL client certificate
+      # option ssl_ca_certs_from_system [Boolean] Use the CA certs from your system's default
+      #   certificate store
+      setting :ssl_ca_certs_from_system, false
+      # option ssl_client_cert [String, nil] SSL client certificate
       setting :ssl_client_cert, nil
-      # option ssl_client_cert_key [String] SSL client certificate password
+      # option ssl_client_cert_key [String, nil] SSL client certificate password
       setting :ssl_client_cert_key, nil
-      # option sasl_gssapi_principal [String] sasl principal
+      # option sasl_gssapi_principal [String, nil] sasl principal
       setting :sasl_gssapi_principal, nil
-      # option sasl_gssapi_keytab [String] sasl keytab
+      # option sasl_gssapi_keytab [String, nil] sasl keytab
       setting :sasl_gssapi_keytab, nil
       # option sasl_plain_authzid [String] The authorization identity to use
       setting :sasl_plain_authzid, ''
-      # option sasl_plain_username [String] The username used to authenticate
+      # option sasl_plain_username [String, nil] The username used to authenticate
       setting :sasl_plain_username, nil
-      # option sasl_plain_password [String] The password used to authenticate
+      # option sasl_plain_password [String, nil] The password used to authenticate
       setting :sasl_plain_password, nil
+      # option sasl_scram_username [String, nil] The username used to authenticate
+      setting :sasl_scram_username, nil
+      # option sasl_scram_password [String, nil] The password used to authenticate
+      setting :sasl_scram_password, nil
+      # option sasl_scram_mechanism [String, nil] Scram mechanism, either 'sha256' or 'sha512'
+      setting :sasl_scram_mechanism, nil
     end
 
     # option producer [Hash] - optional - producer configuration options

data/lib/water_drop/errors.rb

@@ -11,5 +11,8 @@ module WaterDrop
 
     # Raised when we try to send message with invalid optionss
     InvalidMessageOptions = Class.new(BaseError)
+
+    # Raised when want to hook up to an event that is not registered and supported
+    UnregisteredMonitorEvent = Class.new(BaseError)
   end
 end

data/lib/water_drop/instrumentation/listener.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Instrumentation
+    # Default listener that hooks up to our instrumentation and uses its events for logging
+    # It can be removed/replaced or anything without any harm to the Waterdrop flow
+    # @note It is a module as we can use it then as a part of the Karafka framework listener
+    #   as well as we can use it standalone
+    module Listener
+      # Log levels that we use in this particular listener
+      USED_LOG_LEVELS = %i[
+        info
+        error
+      ].freeze
+
+      %i[
+        sync_producer
+        async_producer
+      ].each do |producer_type|
+        error_name = :"on_#{producer_type}_call_error"
+        retry_name = :"on_#{producer_type}_call_retry"
+
+        define_method error_name do |event|
+          options = event[:options]
+          error = event[:error]
+          error "Delivery failure to: #{options} because of #{error}"
+        end
+
+        define_method retry_name do |event|
+          attempts_count = event[:attempts_count]
+          options = event[:options]
+          error = event[:error]
+
+          info "Attempt #{attempts_count} of delivery to: #{options} because of #{error}"
+        end
+
+        module_function error_name
+        module_function retry_name
+        public error_name
+        public retry_name
+      end
+
+      USED_LOG_LEVELS.each do |log_level|
+        define_method log_level do |*args|
+          WaterDrop.logger.send(log_level, *args)
+        end
+
+        module_function log_level
+        private_class_method log_level
+      end
+    end
+  end
+end

data/lib/water_drop/instrumentation/monitor.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  # Namespace for all the things related with WaterDrop instrumentation process
+  module Instrumentation
+    # Monitor is used to hookup external monitoring services to monitor how WaterDrop works
+    # Since it is a pub-sub based on dry-monitor, you can use as many subscribers/loggers at the
+    # same time, which means that you might have for example file logging and newrelic at the same
+    # time
+    # @note This class acts as a singleton because we are only permitted to have single monitor
+    #   per running process (just as logger)
+    class Monitor < Dry::Monitor::Notifications
+      include Singleton
+
+      # List of events that we support in the system and to which a monitor client can hook up
+      # @note The non-error once support timestamp benchmarking
+      BASE_EVENTS = %w[
+        async_producer.call.error
+        async_producer.call.retry
+        sync_producer.call.error
+        sync_producer.call.retry
+      ].freeze
+
+      private_constant :BASE_EVENTS
+
+      # @return [WaterDrop::Instrumentation::Monitor] monitor instance for system instrumentation
+      def initialize
+        super(:waterdrop)
+        BASE_EVENTS.each(&method(:register_event))
+      end
+
+      # Allows us to subscribe to events with a code that will be yielded upon events
+      # @param event_name_or_listener [String, Object] name of the event we want to subscribe to
+      #   or a listener if we decide to go with object listener
+      def subscribe(event_name_or_listener)
+        return super unless event_name_or_listener.is_a?(String)
+        return super if available_events.include?(event_name_or_listener)
+        raise Errors::UnregisteredMonitorEvent, event_name_or_listener
+      end
+
+      # @return [Array<String>] names of available events to which we can subscribe
+      def available_events
+        __bus__.events.keys
+      end
+    end
+  end
+end

data/lib/water_drop/schemas/config.rb

@@ -11,18 +11,8 @@ module WaterDrop
         kafka+ssl
       ].freeze
 
-      # All encryption related options keys
-      ENCRYPTION_OPTIONS_KEYS = %i[
-        ssl_ca_cert
-        ssl_ca_cert_file_path
-        ssl_client_cert
-        ssl_client_cert_key
-        sasl_plain_authzid
-        sasl_plain_username
-        sasl_plain_password
-        sasl_gssapi_principal
-        sasl_gssapi_keytab
-      ].freeze
+      # Available sasl scram mechanism of authentication (plus nil)
+      SASL_SCRAM_MECHANISMS ||= %w[sha256 sha512].freeze
 
       configure do
         config.messages_file = File.join(
@@ -43,13 +33,14 @@ module WaterDrop
       required(:client_id).filled(:str?, format?: Schemas::TOPIC_REGEXP)
       required(:logger).filled
       required(:deliver).filled(:bool?)
+      required(:raise_on_buffer_overflow).filled(:bool?)
 
       required(:kafka).schema do
         required(:seed_brokers).filled { each(:broker_schema?) }
         required(:connect_timeout).filled { (int? | float?) & gt?(0) }
         required(:socket_timeout).filled { (int? | float?) & gt?(0) }
         required(:compression_threshold).filled(:int?, gteq?: 1)
-        optional(:compression_codec).maybe(included_in?: %i[snappy gzip])
+        optional(:compression_codec).maybe(included_in?: %i[snappy gzip lz4])
 
         required(:max_buffer_bytesize).filled(:int?, gt?: 0)
         required(:max_buffer_size).filled(:int?, gt?: 0)
@@ -63,9 +54,27 @@ module WaterDrop
         required(:retry_backoff).filled(:int?, gteq?: 0)
         required(:required_acks).filled(included_in?: [1, 0, -1, :all])
 
-        ENCRYPTION_OPTIONS_KEYS.each do |encryption_attribute|
+        %i[
+          ssl_ca_cert
+          ssl_ca_cert_file_path
+          ssl_client_cert
+          ssl_client_cert_key
+          sasl_gssapi_principal
+          sasl_gssapi_keytab
+          sasl_plain_authzid
+          sasl_plain_username
+          sasl_plain_password
+          sasl_scram_username
+          sasl_scram_password
+        ].each do |encryption_attribute|
           optional(encryption_attribute).maybe(:str?)
         end
+
+        optional(:ssl_ca_certs_from_system).maybe(:bool?)
+
+        # It's not with other encryptions as it has some more rules
+        optional(:sasl_scram_mechanism)
+          .maybe(:str?, included_in?: WaterDrop::Schemas::SASL_SCRAM_MECHANISMS)
       end
     end
   end

data/lib/water_drop/schemas/message_options.rb

@@ -13,6 +13,7 @@ module WaterDrop
       optional(:key).maybe(:str?, :filled?)
       optional(:partition).filled(:int?, gteq?: 0)
       optional(:partition_key).maybe(:str?, :filled?)
+      optional(:create_time).maybe(:time?)
     end
   end
 end

data/lib/water_drop/sync_producer.rb

@@ -10,20 +10,15 @@ module WaterDrop
     # @raise [WaterDrop::Errors::InvalidMessageOptions] raised when message options are
     #   somehow invalid and we cannot perform delivery because of that
     def self.call(message, options)
-      attempts ||= 0
-      attempts += 1
+      attempts_count ||= 0
+      attempts_count += 1
 
       validate!(options)
       return unless WaterDrop.config.deliver
+
       DeliveryBoy.deliver(message, options)
-    rescue Kafka::Error => e
-      if attempts > WaterDrop.config.kafka.max_retries
-        WaterDrop.logger.error e
-        raise e
-      else
-        WaterDrop.logger.warn "Retrying delivery after: #{e}"
-        retry
-      end
+    rescue Kafka::Error => error
+      graceful_attempt?(attempts_count, message, options, error) ? retry : raise(error)
     end
   end
 end

data/lib/water_drop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '1.0.1'
+  VERSION = '1.2.0.beta1'
 end

data/waterdrop.gemspec

@@ -12,15 +12,16 @@ Gem::Specification.new do |spec|
   spec.authors       = ['Maciej Mensfeld']
   spec.email         = %w[maciej@mensfeld.pl]
   spec.homepage      = 'https://github.com/karafka/waterdrop'
-  spec.summary       = ' Kafka messaging made easy! '
+  spec.summary       = 'Kafka messaging made easy!'
   spec.description   = spec.summary
   spec.license       = 'MIT'
 
-  spec.add_dependency 'delivery_boy', '>= 0.2.2'
+  spec.add_dependency 'delivery_boy', '>= 0.2.3'
   spec.add_dependency 'dry-configurable', '~> 0.7'
+  spec.add_dependency 'dry-monitor', '~> 0.1'
   spec.add_dependency 'dry-validation', '~> 0.11'
   spec.add_dependency 'null-logger'
-  spec.add_dependency 'ruby-kafka', '>= 0.5'
+  spec.add_dependency 'ruby-kafka', '>= 0.5.3'
 
   spec.required_ruby_version = '>= 2.2.0'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.2.0.beta1
 platform: ruby
 authors:
 - Maciej Mensfeld
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-06 00:00:00.000000000 Z
+date: 2018-02-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: delivery_boy
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.2
+        version: 0.2.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.2
+        version: 0.2.3
 - !ruby/object:Gem::Dependency
   name: dry-configurable
   requirement: !ruby/object:Gem::Requirement
@@ -38,6 +38,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: dry-monitor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: dry-validation
   requirement: !ruby/object:Gem::Requirement
@@ -72,14 +86,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: 0.5.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: 0.5.3
 description: Kafka messaging made easy!
 email:
 - maciej@mensfeld.pl
@@ -103,6 +117,8 @@ files:
 - lib/water_drop/base_producer.rb
 - lib/water_drop/config.rb
 - lib/water_drop/errors.rb
+- lib/water_drop/instrumentation/listener.rb
+- lib/water_drop/instrumentation/monitor.rb
 - lib/water_drop/schemas/config.rb
 - lib/water_drop/schemas/message_options.rb
 - lib/water_drop/sync_producer.rb
@@ -124,12 +140,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.2.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 2.7.3
 signing_key: 
 specification_version: 4
 summary: Kafka messaging made easy!