waterdrop 2.0.5 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 310a3d7e1a4d0e5825b3a01f59b29c22a9f180c639951763bdf936a23c1119fd
-  data.tar.gz: f6c0c498266ba067201e7983d5bdea7a0aee7810a403be1cd4f4b3d62ab60633
+  metadata.gz: db36e5203312e482d0d98a83d3bfb4fd5e274c342e5b3c63829d6bb5f616b52e
+  data.tar.gz: 6e49baf2f5ebd33aded769899fc0eb7f3c130ac97d9a6156fd491d000fd04306
 SHA512:
-  metadata.gz: 4e486cfa6aa673e008eeaccb8cf920fbb30fce1d23277021d3c6a02e36ee14b8a280e9114b9be778bdb68ba4b07eb2d64371362c454c607edf3c4b57a26a0066
-  data.tar.gz: 50301b9c5a5e67434f46247b5d1a83e4af2577e0f3b8f251a2795bc48aaba8c59135025e606b8143ca57560a2eac6666c530bd5d1b6059ce2e61d008e1eb9385
+  metadata.gz: 426d9f910ca5e24a7e86db1ceec672f895186ace830554708e8741d21bb83f929c9278ed5018bff6468b354523ac7cc790cba0b91b8c13dd2bb28e58f8a52dc6
+  data.tar.gz: b3871df6868ce287daf0765f522031cfdd566b96d0e5f8bf3ca1cd9724a7a68d5b3e4e1a33e0bdcc9b65063150d48cfc213e19348d19a6268a309b1c69dcab50

data/.github/workflows/ci.yml

@@ -1,5 +1,7 @@
 name: ci
 
+concurrency: ci-${{ github.ref }}
+
 on:
   pull_request:
   push:
@@ -14,12 +16,11 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
+          - '3.1'
           - '3.0'
           - '2.7'
-          - '2.6'
-          - 'jruby-9.3.1.0'
         include:
-          - ruby: '3.0'
+          - ruby: '3.1'
             coverage: 'true'
     steps:
       - uses: actions/checkout@v2
@@ -59,7 +60,7 @@ jobs:
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 3.0
+          ruby-version: 3.1
       - name: Install latest bundler
         run: gem install bundler --no-document
       - name: Install Diffend plugin

data/.ruby-version

@@ -1 +1 @@
-3.0.2
+3.1.0

data/CHANGELOG.md

@@ -1,5 +1,30 @@
 # WaterDrop changelog
 
+## 2.2.0 (2022-02-18)
+- Add Datadog listener for metrics + errors publishing
+- Add Datadog example dashboard template
+- Update Readme to show Dd instrumentation usage
+- Align the directory namespace convention with gem name (waterdrop => WaterDrop)
+- Introduce a common base for validation contracts
+- Drop CI support for ruby 2.6
+- Require all `kafka` settings to have symbol keys (compatibility with Karafka 2.0 and rdkafka)
+
+## 2.1.0 (2022-01-03)
+- Ruby 3.1 support
+- Change the error notification key from `error.emitted` to `error.occurred`.
+- Normalize error tracking and make all the places publish errors into the same notification endpoint (`error.occurred`).
+- Start semantic versioning WaterDrop.
+
+## 2.0.7 (2021-12-03)
+- Source code metadata url added to the gemspec
+- Replace `:producer` with `:producer_id` in events and update `StdoutListener` accordingly. This change aligns all the events in terms of not publishing the whole producer object in the events.
+- Add `error.emitted` into the `StdoutListener`.
+- Enable `StdoutLogger` in specs for additional integration coverage.
+
+## 2.0.6 (2021-12-01)
+- #218 - Fixes a case, where dispatch of callbacks the same moment a new producer was created could cause a concurrency issue in the manager.
+- Fix some unstable specs.
+
 ## 2.0.5 (2021-11-28)
 
 ### Bug fixes

data/Gemfile

@@ -6,8 +6,6 @@ plugin 'diffend'
 
 gemspec
 
-gem 'rdkafka'
-
 group :development do
   gem 'byebug'
 end

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.0.5)
+    waterdrop (2.2.0)
       concurrent-ruby (>= 1.1)
       dry-configurable (~> 0.13)
       dry-monitor (~> 0.5)
@@ -12,17 +12,16 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.1.4.1)
+    activesupport (7.0.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
-      zeitwerk (~> 2.3)
     byebug (11.1.3)
     concurrent-ruby (1.1.9)
-    diff-lcs (1.4.4)
+    diff-lcs (1.5.0)
     docile (1.4.0)
-    dry-configurable (0.13.0)
+    dry-configurable (0.14.0)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.6)
     dry-container (0.9.0)
@@ -34,7 +33,7 @@ GEM
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.5, >= 0.5)
     dry-inflector (0.2.1)
-    dry-initializer (3.0.4)
+    dry-initializer (3.1.1)
     dry-logic (1.2.0)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.5, >= 0.5)
@@ -63,29 +62,29 @@ GEM
       dry-schema (~> 1.8, >= 1.8.0)
     factory_bot (6.2.0)
       activesupport (>= 5.0.0)
-    ffi (1.15.4)
-    i18n (1.8.11)
+    ffi (1.15.5)
+    i18n (1.9.1)
       concurrent-ruby (~> 1.0)
     mini_portile2 (2.7.1)
-    minitest (5.14.4)
+    minitest (5.15.0)
     rake (13.0.6)
-    rdkafka (0.11.0)
+    rdkafka (0.11.1)
       ffi (~> 1.15)
-      mini_portile2 (~> 2.7)
+      mini_portile2 (~> 2.6)
       rake (> 12)
-    rspec (3.10.0)
-      rspec-core (~> 3.10.0)
-      rspec-expectations (~> 3.10.0)
-      rspec-mocks (~> 3.10.0)
-    rspec-core (3.10.1)
-      rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.1)
+    rspec (3.11.0)
+      rspec-core (~> 3.11.0)
+      rspec-expectations (~> 3.11.0)
+      rspec-mocks (~> 3.11.0)
+    rspec-core (3.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-expectations (3.11.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.2)
+      rspec-support (~> 3.11.0)
+    rspec-mocks (3.11.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-support (3.10.3)
+      rspec-support (~> 3.11.0)
+    rspec-support (3.11.0)
     simplecov (0.21.2)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
@@ -94,19 +93,17 @@ GEM
     simplecov_json_formatter (0.1.3)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    zeitwerk (2.5.1)
+    zeitwerk (2.5.4)
 
 PLATFORMS
-  x86_64-darwin
   x86_64-linux
 
 DEPENDENCIES
   byebug
   factory_bot
-  rdkafka
   rspec
   simplecov
   waterdrop!
 
 BUNDLED WITH
-   2.2.31
+   2.3.7

data/README.md

@@ -2,7 +2,7 @@
 
 **Note**: Documentation presented here refers to WaterDrop `2.0.0`.
 
-WaterDrop `2.0` does **not** work with Karafka `1.*` and aims to either work as a standalone producer outside of Karafka `1.*` ecosystem or as a part of not yet released Karafka `2.0.*`.
+WaterDrop `2.0` does **not** work with Karafka `1.*` and aims to either work as a standalone producer outside of Karafka `1.*` ecosystem or as a part of soon to be released Karafka `2.0.*`.
 
 Please refer to [this](https://github.com/karafka/waterdrop/tree/1.4) branch and its documentation for details about WaterDrop `1.*` usage.
 
@@ -36,16 +36,13 @@ It:
 - [Instrumentation](#instrumentation)
   * [Usage statistics](#usage-statistics)
   * [Error notifications](#error-notifications)
+  * [Datadog and StatsD integration](#datadog-and-statsd-integration)
   * [Forking and potential memory problems](#forking-and-potential-memory-problems)
 - [Note on contributions](#note-on-contributions)
 
 ## Installation
 
-```ruby
-gem install waterdrop
-```
-
-or add this to your Gemfile:
+Add this to your Gemfile:
 
 ```ruby
 gem 'waterdrop'
@@ -201,7 +198,7 @@ producer.setup do |config|
   config.kafka = {
     'bootstrap.servers': 'localhost:9092',
     # Accumulate messages for at most 10 seconds
-    'queue.buffering.max.ms' => 10_000
+    'queue.buffering.max.ms': 10_000
   }
 end
 
@@ -288,9 +285,40 @@ producer.close
 
 Note: The metrics returned may not be completely consistent between brokers, toppars and totals, due to the internal asynchronous nature of librdkafka. E.g., the top level tx total may be less than the sum of the broker tx values which it represents.
 
+### Datadog and StatsD integration
+
+WaterDrop comes with (optional) full Datadog and StatsD integration that you can use. To use it:
+
+```ruby
+# require datadog/statsd and the listener as it is not loaded by default
+require 'datadog/statsd'
+require 'waterdrop/instrumentation/vendors/datadog/listener'
+
+# initialize your producer with statistics.interval.ms enabled so the metrics are published
+producer = WaterDrop::Producer.new do |config|
+  config.deliver = true
+  config.kafka = {
+    'bootstrap.servers': 'localhost:9092',
+    'statistics.interval.ms': 1_000
+  }
+end
+
+# initialize the listener with statsd client
+listener = ::WaterDrop::Instrumentation::Vendors::Datadog::Listener.new do |config|
+  config.client = Datadog::Statsd.new('localhost', 8125)
+  # Publish host as a tag alongside the rest of tags
+  config.default_tags = ["host:#{Socket.gethostname}"]
+end
+
+# Subscribe with your listener to your producer and you should be ready to go!
+producer.monitor.subscribe(listener)
+```
+
+You can also find a ready to import DataDog dashboard configuration file that you can use to monitor all of your producer.
+
 ### Error notifications
 
-Aside from errors related to publishing messages like `buffer.flushed_async.error`, WaterDrop allows you to listen to errors that occur in its internal background threads. Things like reconnecting to Kafka upon network errors and others unrelated to publishing messages are all available under `error.emitted` notification key. You can subscribe to this event to ensure your setup is healthy and without any problems that would otherwise go unnoticed as long as messages are delivered.
+WaterDrop allows you to listen to all errors that occur while producing messages and in its internal background threads. Things like reconnecting to Kafka upon network errors and others unrelated to publishing messages are all available under `error.occurred` notification key. You can subscribe to this event to ensure your setup is healthy and without any problems that would otherwise go unnoticed as long as messages are delivered.
 
 ```ruby
 producer = WaterDrop::Producer.new do |config|
@@ -298,10 +326,10 @@ producer = WaterDrop::Producer.new do |config|
   config.kafka = { 'bootstrap.servers': 'localhost:9090' }
 end
 
-producer.monitor.subscribe('error.emitted') do |event|
+producer.monitor.subscribe('error.occurred') do |event|
   error = event[:error]
 
-  p "Internal error occurred: #{error}"
+  p "WaterDrop error occurred: #{error}"
 end
 
 # Run this code without Kafka cluster
@@ -313,9 +341,9 @@ end
 
 # After you stop your Kafka cluster, you will see a lot of those:
 #
-# Internal error occurred: Local: Broker transport failure (transport)
+# WaterDrop error occurred: Local: Broker transport failure (transport)
 #
-# Internal error occurred: Local: Broker transport failure (transport)
+# WaterDrop error occurred: Local: Broker transport failure (transport)
 ```
 
 ### Forking and potential memory problems

data/config/errors.yml

@@ -4,3 +4,4 @@ en:
       invalid_key_type: all keys need to be of type String
       invalid_value_type: all values need to be of type String
       max_payload_size: is more than `max_payload_size` config value
+      kafka_key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols

data/lib/{water_drop → waterdrop}/config.rb

@@ -9,7 +9,7 @@ module WaterDrop
 
     # Defaults for kafka settings, that will be overwritten only if not present already
     KAFKA_DEFAULTS = {
-      'client.id' => 'waterdrop'
+      'client.id': 'waterdrop'
     }.freeze
 
     private_constant :KAFKA_DEFAULTS
@@ -17,7 +17,8 @@ module WaterDrop
     # WaterDrop options
     #
     # option [String] id of the producer. This can be helpful when building producer specific
-    #   instrumentation or loggers. It is not the kafka producer id
+    #   instrumentation or loggers. It is not the kafka client id. It is an id that should be
+    #   unique for each of the producers
     setting(
       :id,
       default: false,
@@ -62,7 +63,8 @@ module WaterDrop
         yield(config)
 
         merge_kafka_defaults!(config)
-        validate!(config.to_h)
+
+        Contracts::Config.new.validate!(config.to_h, Errors::ConfigurationInvalidError)
 
         ::Rdkafka::Config.logger = config.logger
       end
@@ -81,16 +83,5 @@ module WaterDrop
         config.kafka[key] = value
       end
     end
-
-    # Validates the configuration and if anything is wrong, will raise an exception
-    # @param config_hash [Hash] config hash with setup details
-    # @raise [WaterDrop::Errors::ConfigurationInvalidError] raised when something is wrong with
-    #   the configuration
-    def validate!(config_hash)
-      result = Contracts::Config.new.call(config_hash)
-      return true if result.success?
-
-      raise Errors::ConfigurationInvalidError, result.errors.to_h
-    end
   end
 end

data/lib/waterdrop/contracts/base.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Contracts
+    # Base for all the contracts in WaterDrop
+    class Base < Dry::Validation::Contract
+      config.messages.load_paths << File.join(WaterDrop.gem_root, 'config', 'errors.yml')
+
+      # @param data [Hash] data for validation
+      # @param error_class [Class] error class that should be used when validation fails
+      # @return [Boolean] true
+      # @raise [StandardError] any error provided in the error_class that inherits from the
+      #   standard error
+      def validate!(data, error_class)
+        result = call(data)
+
+        return true if result.success?
+
+        raise error_class, result.errors.to_h
+      end
+    end
+  end
+end

data/lib/{water_drop → waterdrop}/contracts/config.rb

@@ -3,12 +3,7 @@
 module WaterDrop
   module Contracts
     # Contract with validation rules for WaterDrop configuration details
-    class Config < Dry::Validation::Contract
-      # Ensure valid format of each seed broker so that rdkafka doesn't fail silently
-      SEED_BROKER_FORMAT_REGEXP = %r{\A([^:/,]+:[0-9]+)(,[^:/,]+:[0-9]+)*\z}.freeze
-
-      private_constant :SEED_BROKER_FORMAT_REGEXP
-
+    class Config < Base
       params do
         required(:id).filled(:str?)
         required(:logger).filled
@@ -16,9 +11,18 @@ module WaterDrop
         required(:max_payload_size).filled(:int?, gteq?: 1)
         required(:max_wait_timeout).filled(:number?, gteq?: 0)
         required(:wait_timeout).filled(:number?, gt?: 0)
+        required(:kafka).filled(:hash?)
+      end
+
+      # rdkafka allows both symbols and strings as keys for config but then casts them to strings
+      # This can be confusing, so we expect all keys to be symbolized
+      rule(:kafka) do
+        next unless value.is_a?(Hash)
+
+        value.each_key do |key|
+          next if key.is_a?(Symbol)
 
-        required(:kafka).schema do
-          required(:'bootstrap.servers').filled(:str?, format?: SEED_BROKER_FORMAT_REGEXP)
+          key(:"kafka.#{key}").failure(:kafka_key_must_be_a_symbol)
         end
       end
     end

data/lib/{water_drop → waterdrop}/contracts/message.rb

@@ -4,17 +4,15 @@ module WaterDrop
   module Contracts
     # Contract with validation rules for validating that all the message options that
     # we provide to producer ale valid and usable
-    class Message < Dry::Validation::Contract
+    class Message < Base
       # Regex to check that topic has a valid format
-      TOPIC_REGEXP = /\A(\w|-|\.)+\z/.freeze
+      TOPIC_REGEXP = /\A(\w|-|\.)+\z/
 
       # Checks, that the given value is a string
       STRING_ASSERTION = ->(value) { value.is_a?(String) }.to_proc
 
       private_constant :TOPIC_REGEXP, :STRING_ASSERTION
 
-      config.messages.load_paths << File.join(WaterDrop.gem_root, 'config', 'errors.yml')
-
       option :max_payload_size
 
       params do

data/lib/{water_drop → waterdrop}/instrumentation/callbacks/error.rb

@@ -17,16 +17,17 @@ module WaterDrop
         # Runs the instrumentation monitor with error
         # @param client_name [String] rdkafka client name
         # @param error [Rdkafka::Error] error that occurred
-        # @note If will only instrument on errors of the client of our producer
+        # @note It will only instrument on errors of the client of our producer
         def call(client_name, error)
           # Emit only errors related to our client
           # Same as with statistics (mor explanation there)
           return unless @client_name == client_name
 
           @monitor.instrument(
-            'error.emitted',
+            'error.occurred',
+            error: error,
             producer_id: @producer_id,
-            error: error
+            type: 'librdkafka.error'
           )
         end
       end

data/lib/{water_drop → waterdrop}/instrumentation/callbacks_manager.rb

@@ -13,8 +13,12 @@ module WaterDrop
       # Invokes all the callbacks registered one after another
       #
       # @param args [Object] any args that should go to the callbacks
+      # @note We do not use `#each_value` here on purpose. With it being used, we cannot dispatch
+      #   callbacks and add new at the same time. Since we don't know when and in what thread
+      #   things are going to be added to the manager, we need to extract values into an array and
+      #   run it. That way we can add new things the same time.
       def call(*args)
-        @callbacks.each_value { |a| a.call(*args) }
+        @callbacks.values.each { |callback| callback.call(*args) }
       end
 
       # Adds a callback to the manager

data/lib/{water_drop → waterdrop}/instrumentation/monitor.rb

@@ -24,13 +24,11 @@ module WaterDrop
         messages.buffered
 
         buffer.flushed_async
-        buffer.flushed_async.error
         buffer.flushed_sync
-        buffer.flushed_sync.error
 
         statistics.emitted
 
-        error.emitted
+        error.occurred
       ].freeze
 
       private_constant :EVENTS

data/lib/{water_drop → waterdrop}/instrumentation/stdout_listener.rb

@@ -51,7 +51,7 @@ module WaterDrop
         message = event[:message]
 
         info(event, "Buffering of a message to '#{message[:topic]}' topic")
-        debug(event, [message, event[:producer].messages.size])
+        debug(event, [message])
       end
 
       # @param event [Dry::Events::Event] event that happened with the details
@@ -59,7 +59,7 @@ module WaterDrop
         messages = event[:messages]
 
         info(event, "Buffering of #{messages.size} messages")
-        debug(event, [messages, event[:producer].messages.size])
+        debug(event, [messages, messages.size])
       end
 
       # @param event [Dry::Events::Event] event that happened with the details
@@ -70,15 +70,6 @@ module WaterDrop
         debug(event, messages)
       end
 
-      # @param event [Dry::Events::Event] event that happened with the details
-      def on_buffer_flushed_async_error(event)
-        messages = event[:messages]
-        error = event[:error]
-
-        error(event, "Async flushing of #{messages.size} failed due to: #{error}")
-        debug(event, messages)
-      end
-
       # @param event [Dry::Events::Event] event that happened with the details
       def on_buffer_flushed_sync(event)
         messages = event[:messages]
@@ -87,19 +78,19 @@ module WaterDrop
         debug(event, messages)
       end
 
-      # @param event [Dry::Events::Event] event that happened with the details
-      def on_buffer_flushed_sync_error(event)
-        messages = event[:dispatched]
-        error = event[:error]
-
-        error(event, "Sync flushing of #{messages.size} failed due to: #{error}")
-        debug(event, messages)
-      end
-
       # @param event [Dry::Events::Event] event that happened with the details
       def on_producer_closed(event)
         info event, 'Closing producer'
-        debug event, event[:producer].messages.size
+        debug event, ''
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the error details
+      def on_error_occurred(event)
+        error = event[:error]
+        type = event[:type]
+
+        error(event, "Error occurred: #{error} - #{type}")
+        debug(event, '')
       end
 
       private
@@ -107,19 +98,19 @@ module WaterDrop
       # @param event [Dry::Events::Event] event that happened with the details
       # @param log_message [String] message we want to publish
       def debug(event, log_message)
-        @logger.debug("[#{event[:producer].id}] #{log_message}")
+        @logger.debug("[#{event[:producer_id]}] #{log_message}")
       end
 
       # @param event [Dry::Events::Event] event that happened with the details
       # @param log_message [String] message we want to publish
       def info(event, log_message)
-        @logger.info("[#{event[:producer].id}] #{log_message} took #{event[:time]} ms")
+        @logger.info("[#{event[:producer_id]}] #{log_message} took #{event[:time]} ms")
       end
 
       # @param event [Dry::Events::Event] event that happened with the details
       # @param log_message [String] message we want to publish
       def error(event, log_message)
-        @logger.error("[#{event[:producer].id}] #{log_message}")
+        @logger.error("[#{event[:producer_id]}] #{log_message}")
       end
     end
   end

data/lib/waterdrop/instrumentation/vendors/datadog/dashboard.json

@@ -0,0 +1 @@
+{"title":"WaterDrop producer example dashboard","description":"This dashboard include example setup for monitoring activity of your WaterDrop producer","widgets":[{"id":243951318,"definition":{"title":"Messages produced","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"produced sync","formula":"query1"},{"alias":"produced async","formula":"query2"},{"alias":"flushed sync","formula":"query3"},{"alias":"flushed async","formula":"query4"},{"alias":"acknowledged","formula":"query5"}],"response_format":"timeseries","queries":[{"query":"sum:waterdrop.produced_sync{*}.as_count()","data_source":"metrics","name":"query1"},{"query":"sum:waterdrop.produced_async{*}.as_count()","data_source":"metrics","name":"query2"},{"query":"sum:waterdrop.flushed_sync{*}.as_count()","data_source":"metrics","name":"query3"},{"query":"sum:waterdrop.flushed_async{*}.as_count()","data_source":"metrics","name":"query4"},{"query":"sum:waterdrop.acknowledged{*}.as_count()","data_source":"metrics","name":"query5"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"include_zero":true,"scale":"linear","label":"","min":"auto","max":"auto"}}},{"id":1979626566852990,"definition":{"title":"Messages buffer size","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"max","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"avg:waterdrop.buffer.size.max{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]}},{"id":243951221,"definition":{"title":"Kafka broker API calls","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"API calls","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"sum:waterdrop.calls{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"include_zero":true,"scale":"linear","label":"","min":"auto","max":"auto"}}},{"id":243951952,"definition":{"title":"Producer queue size","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Queue size average","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"max:waterdrop.queue.size.avg{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"},{"formulas":[{"alias":"Queue size max","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"max:waterdrop.queue.size.max{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"include_zero":true,"scale":"linear","label":"","min":"auto","max":"auto"}}},{"id":243951263,"definition":{"title":"Producer queue latency","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Average latency","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"avg:waterdrop.queue.latency.avg{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"},{"formulas":[{"alias":"Latency p95","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"avg:waterdrop.queue.latency.p95{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"},{"formulas":[{"alias":"Latency p99","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"avg:waterdrop.queue.latency.p99{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"include_zero":true,"scale":"linear","label":"","min":"auto","max":"auto"}}},{"id":243951276,"definition":{"title":"Producer network latency","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Average latency","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"avg:waterdrop.request_size.avg{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"},{"formulas":[{"alias":"Latency p95","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"avg:waterdrop.network.latency.p95{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"},{"formulas":[{"alias":"Latency p99","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"avg:waterdrop.network.latency.p99{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"include_zero":true,"scale":"linear","label":"","min":"auto","max":"auto"}}},{"id":243954928,"definition":{"title":"Producer errors","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"response_format":"timeseries","queries":[{"query":"sum:waterdrop.error_occurred{*}.as_count()","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"include_zero":true,"scale":"linear","label":"","min":"auto","max":"auto"}}}],"template_variables":[],"layout_type":"ordered","is_read_only":false,"notify_list":[],"reflow_type":"auto","id":"rnr-kgh-dna"}

data/lib/waterdrop/instrumentation/vendors/datadog/listener.rb

@@ -0,0 +1,197 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Instrumentation
+    # Namespace for vendor specific instrumentation
+    module Vendors
+      # Datadog specific instrumentation
+      module Datadog
+        # Listener that can be used to subscribe to WaterDrop producer to receive stats via StatsD
+        # and/or Datadog
+        #
+        # @note You need to setup the `dogstatsd-ruby` client and assign it
+        class Listener
+          include Dry::Configurable
+
+          # Value object for storing a single rdkafka metric publishing details
+          RdKafkaMetric = Struct.new(:type, :scope, :name, :key_location)
+
+          # Namespace under which the DD metrics should be published
+          setting :namespace, default: 'waterdrop', reader: true
+
+          # Datadog client that we should use to publish the metrics
+          setting :client, reader: true
+
+          # Default tags we want to publish (for example hostname)
+          # Format as followed (example for hostname): `["host:#{Socket.gethostname}"]`
+          setting :default_tags, default: [], reader: true
+
+          # All the rdkafka metrics we want to publish
+          #
+          # By default we publish quite a lot so this can be tuned
+          # Note, that the once with `_d` come from WaterDrop, not rdkafka or Kafka
+          setting :rd_kafka_metrics, reader: true, default: [
+            # Client metrics
+            RdKafkaMetric.new(:count, :root, 'calls', 'tx_d'),
+            RdKafkaMetric.new(:histogram, :root, 'queue.size', 'msg_cnt_d'),
+
+            # Broker metrics
+            RdKafkaMetric.new(:count, :brokers, 'deliver.attempts', 'txretries_d'),
+            RdKafkaMetric.new(:count, :brokers, 'deliver.errors', 'txerrs_d'),
+            RdKafkaMetric.new(:count, :brokers, 'receive.errors', 'rxerrs_d'),
+            RdKafkaMetric.new(:gauge, :brokers, 'queue.latency.avg', %w[outbuf_latency avg]),
+            RdKafkaMetric.new(:gauge, :brokers, 'queue.latency.p95', %w[outbuf_latency p95]),
+            RdKafkaMetric.new(:gauge, :brokers, 'queue.latency.p99', %w[outbuf_latency p99]),
+            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.avg', %w[rtt avg]),
+            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.p95', %w[rtt p95]),
+            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.p99', %w[rtt p99])
+          ].freeze
+
+          # @param block [Proc] configuration block
+          def initialize(&block)
+            setup(&block) if block
+          end
+
+          # @param block [Proc] configuration block
+          # @note We define this alias to be consistent with `WaterDrop#setup`
+          def setup(&block)
+            configure(&block)
+          end
+
+          # Hooks up to WaterDrop instrumentation for emitted statistics
+          #
+          # @param event [Dry::Events::Event]
+          def on_statistics_emitted(event)
+            statistics = event[:statistics]
+
+            rd_kafka_metrics.each do |metric|
+              report_metric(metric, statistics)
+            end
+          end
+
+          # Increases the errors count by 1
+          #
+          # @param _event [Dry::Events::Event]
+          def on_error_occurred(_event)
+            client.count(
+              namespaced_metric('error_occurred'),
+              1,
+              tags: default_tags
+            )
+          end
+
+          # Increases acknowledged messages counter
+          # @param _event [Dry::Events::Event]
+          def on_message_acknowledged(_event)
+            client.increment(
+              namespaced_metric('acknowledged'),
+              tags: default_tags
+            )
+          end
+
+          %i[
+            produced_sync
+            produced_async
+          ].each do |event_scope|
+            class_eval <<~METHODS, __FILE__, __LINE__ + 1
+              # @param event [Dry::Events::Event]
+              def on_message_#{event_scope}(event)
+                report_message(event[:message][:topic], :#{event_scope})
+              end
+
+              # @param event [Dry::Events::Event]
+              def on_messages_#{event_scope}(event)
+                event[:messages].each do |message|
+                  report_message(message[:topic], :#{event_scope})
+                end
+              end
+            METHODS
+          end
+
+          # Reports the buffer usage when anything is added to the buffer
+          %i[
+            message_buffered
+            messages_buffered
+          ].each do |event_scope|
+            class_eval <<~METHODS, __FILE__, __LINE__ + 1
+              # @param event [Dry::Events::Event]
+              def on_#{event_scope}(event)
+                client.histogram(
+                  namespaced_metric('buffer.size'),
+                  event[:buffer].size,
+                  tags: default_tags
+                )
+              end
+            METHODS
+          end
+
+          # Events that support many messages only
+          # Reports data flushing operation (production from the buffer)
+          %i[
+            flushed_sync
+            flushed_async
+          ].each do |event_scope|
+            class_eval <<~METHODS, __FILE__, __LINE__ + 1
+              # @param event [Dry::Events::Event]
+              def on_buffer_#{event_scope}(event)
+                event[:messages].each do |message|
+                  report_message(message[:topic], :#{event_scope})
+                end
+              end
+            METHODS
+          end
+
+          private
+
+          # Report that a message has been produced to a topic.
+          # @param topic [String] Kafka topic
+          # @param method_name [Symbol] method from which this message operation comes
+          def report_message(topic, method_name)
+            client.increment(
+              namespaced_metric(method_name),
+              tags: default_tags + ["topic:#{topic}"]
+            )
+          end
+
+          # Wraps metric name in listener's namespace
+          # @param metric_name [String] RdKafkaMetric name
+          # @return [String]
+          def namespaced_metric(metric_name)
+            "#{namespace}.#{metric_name}"
+          end
+
+          # Reports a given metric statistics to Datadog
+          # @param metric [RdKafkaMetric] metric value object
+          # @param statistics [Hash] hash with all the statistics emitted
+          def report_metric(metric, statistics)
+            case metric.scope
+            when :root
+              client.public_send(
+                metric.type,
+                namespaced_metric(metric.name),
+                statistics.fetch(*metric.key_location),
+                tags: default_tags
+              )
+            when :brokers
+              statistics.fetch('brokers').each_value do |broker_statistics|
+                # Skip bootstrap nodes
+                # Bootstrap nodes have nodeid -1, other nodes have positive
+                # node ids
+                next if broker_statistics['nodeid'] == -1
+
+                client.public_send(
+                  metric.type,
+                  namespaced_metric(metric.name),
+                  broker_statistics.dig(*metric.key_location),
+                  tags: default_tags + ["broker:#{broker_statistics['nodename']}"]
+                )
+              end
+            else
+              raise ArgumentError, metric.scope
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/{water_drop → waterdrop}/instrumentation.rb

@@ -2,7 +2,6 @@
 
 module WaterDrop
   # Namespace for all the things related with WaterDrop instrumentation process
-  # @note We do not
   module Instrumentation
     class << self
       # Builds a manager for statistics callbacks

data/lib/{water_drop → waterdrop}/producer/async.rb

@@ -19,7 +19,7 @@ module WaterDrop
 
         @monitor.instrument(
           'message.produced_async',
-          producer: self,
+          producer_id: id,
           message: message
         ) { client.produce(**message) }
       end
@@ -40,7 +40,7 @@ module WaterDrop
 
         @monitor.instrument(
           'messages.produced_async',
-          producer: self,
+          producer_id: id,
           messages: messages
         ) do
           messages.map { |message| client.produce(**message) }

data/lib/{water_drop → waterdrop}/producer/buffer.rb

@@ -23,8 +23,9 @@ module WaterDrop
 
         @monitor.instrument(
           'message.buffered',
-          producer: self,
-          message: message
+          producer_id: id,
+          message: message,
+          buffer: @messages
         ) { @messages << message }
       end
 
@@ -40,8 +41,9 @@ module WaterDrop
 
         @monitor.instrument(
           'messages.buffered',
-          producer: self,
-          messages: messages
+          producer_id: id,
+          messages: messages,
+          buffer: @messages
         ) do
           messages.each { |message| @messages << message }
           messages
@@ -56,7 +58,7 @@ module WaterDrop
 
         @monitor.instrument(
           'buffer.flushed_async',
-          producer: self,
+          producer_id: id,
           messages: @messages
         ) { flush(false) }
       end
@@ -69,7 +71,7 @@ module WaterDrop
 
         @monitor.instrument(
           'buffer.flushed_sync',
-          producer: self,
+          producer_id: id,
           messages: @messages
         ) { flush(true) }
       end
@@ -103,8 +105,13 @@ module WaterDrop
           )
         end
       rescue *RESCUED_ERRORS => e
-        key = sync ? 'buffer.flushed_sync.error' : 'buffer.flush_async.error'
-        @monitor.instrument(key, producer: self, error: e, dispatched: dispatched)
+        @monitor.instrument(
+          'error.occurred',
+          error: e,
+          producer_id: id,
+          dispatched: dispatched,
+          type: sync ? 'buffer.flushed_sync.error' : 'buffer.flush_async.error'
+        )
 
         raise Errors::FlushFailureError.new(dispatched)
       end

data/lib/{water_drop → waterdrop}/producer/sync.rb

@@ -21,7 +21,7 @@ module WaterDrop
 
         @monitor.instrument(
           'message.produced_sync',
-          producer: self,
+          producer_id: id,
           message: message
         ) do
           client
@@ -49,7 +49,7 @@ module WaterDrop
         ensure_active!
         messages.each { |message| validate_message!(message) }
 
-        @monitor.instrument('messages.produced_sync', producer: self, messages: messages) do
+        @monitor.instrument('messages.produced_sync', producer_id: id, messages: messages) do
           messages
             .map { |message| client.produce(**message) }
             .map! do |handler|

data/lib/{water_drop → waterdrop}/producer.rb

@@ -106,7 +106,7 @@ module WaterDrop
 
         @monitor.instrument(
           'producer.closed',
-          producer: self
+          producer_id: id
         ) do
           @status.closing!
 
@@ -150,13 +150,7 @@ module WaterDrop
     # @param message [Hash] message we want to send
     # @raise [Karafka::Errors::MessageInvalidError]
     def validate_message!(message)
-      result = @contract.call(message)
-      return if result.success?
-
-      raise Errors::MessageInvalidError, [
-        result.errors.to_h,
-        message
-      ]
+      @contract.validate!(message, Errors::MessageInvalidError)
     end
   end
 end

data/lib/{water_drop → waterdrop}/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '2.0.5'
+  VERSION = '2.2.0'
 end

data/lib/waterdrop.rb

@@ -1,4 +1,37 @@
 # frozen_string_literal: true
 
-# This file is used as a compatibility step
-require 'water_drop'
+# External components
+# delegate should be removed because we don't need it, we just add it because of ruby-kafka
+%w[
+  concurrent/array
+  dry-configurable
+  dry/monitor/notifications
+  dry-validation
+  rdkafka
+  json
+  zeitwerk
+  securerandom
+].each { |lib| require lib }
+
+# WaterDrop library
+module WaterDrop
+  class << self
+    # @return [String] root path of this gem
+    def gem_root
+      Pathname.new(File.expand_path('..', __dir__))
+    end
+  end
+end
+
+loader = Zeitwerk::Loader.for_gem
+loader.inflector.inflect('waterdrop' => 'WaterDrop')
+# Do not load vendors instrumentation components. Those need to be required manually if needed
+loader.ignore("#{__dir__}/waterdrop/instrumentation/vendors/**/*.rb")
+loader.setup
+loader.eager_load
+
+# Rdkafka uses a single global callback for things. We bypass that by injecting a manager for
+# each callback type. Callback manager allows us to register more than one callback
+# @note Those managers are also used by Karafka for consumer related statistics
+Rdkafka::Config.statistics_callback = WaterDrop::Instrumentation.statistics_callbacks
+Rdkafka::Config.error_callback = WaterDrop::Instrumentation.error_callbacks

data/waterdrop.gemspec

@@ -3,7 +3,7 @@
 lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
-require 'water_drop/version'
+require 'waterdrop/version'
 
 Gem::Specification.new do |spec|
   spec.name          = 'waterdrop'
@@ -11,7 +11,7 @@ Gem::Specification.new do |spec|
   spec.platform      = Gem::Platform::RUBY
   spec.authors       = ['Maciej Mensfeld']
   spec.email         = %w[maciej@mensfeld.pl]
-  spec.homepage      = 'https://github.com/karafka/waterdrop'
+  spec.homepage      = 'https://karafka.io'
   spec.summary       = 'Kafka messaging made easy!'
   spec.description   = spec.summary
   spec.license       = 'MIT'
@@ -23,7 +23,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'rdkafka', '>= 0.10'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
-  spec.required_ruby_version = '>= 2.6.0'
+  spec.required_ruby_version = '>= 2.7'
 
   if $PROGRAM_NAME.end_with?('gem')
     spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
@@ -33,4 +33,9 @@ Gem::Specification.new do |spec|
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = %w[lib]
+
+  spec.metadata = {
+    'source_code_uri' => 'https://github.com/karafka/waterdrop',
+    'rubygems_mfa_required' => 'true'
+  }
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 2.0.5
+  version: 2.2.0
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -34,7 +34,7 @@ cert_chain:
   R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
   pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
   -----END CERTIFICATE-----
-date: 2021-11-28 00:00:00.000000000 Z
+date: 2022-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -142,37 +142,41 @@ files:
 - certs/mensfeld.pem
 - config/errors.yml
 - docker-compose.yml
-- lib/water_drop.rb
-- lib/water_drop/config.rb
-- lib/water_drop/contracts.rb
-- lib/water_drop/contracts/config.rb
-- lib/water_drop/contracts/message.rb
-- lib/water_drop/errors.rb
-- lib/water_drop/instrumentation.rb
-- lib/water_drop/instrumentation/callbacks/delivery.rb
-- lib/water_drop/instrumentation/callbacks/error.rb
-- lib/water_drop/instrumentation/callbacks/statistics.rb
-- lib/water_drop/instrumentation/callbacks/statistics_decorator.rb
-- lib/water_drop/instrumentation/callbacks_manager.rb
-- lib/water_drop/instrumentation/monitor.rb
-- lib/water_drop/instrumentation/stdout_listener.rb
-- lib/water_drop/patches/rdkafka/bindings.rb
-- lib/water_drop/patches/rdkafka/producer.rb
-- lib/water_drop/producer.rb
-- lib/water_drop/producer/async.rb
-- lib/water_drop/producer/buffer.rb
-- lib/water_drop/producer/builder.rb
-- lib/water_drop/producer/dummy_client.rb
-- lib/water_drop/producer/status.rb
-- lib/water_drop/producer/sync.rb
-- lib/water_drop/version.rb
 - lib/waterdrop.rb
+- lib/waterdrop/config.rb
+- lib/waterdrop/contracts.rb
+- lib/waterdrop/contracts/base.rb
+- lib/waterdrop/contracts/config.rb
+- lib/waterdrop/contracts/message.rb
+- lib/waterdrop/errors.rb
+- lib/waterdrop/instrumentation.rb
+- lib/waterdrop/instrumentation/callbacks/delivery.rb
+- lib/waterdrop/instrumentation/callbacks/error.rb
+- lib/waterdrop/instrumentation/callbacks/statistics.rb
+- lib/waterdrop/instrumentation/callbacks/statistics_decorator.rb
+- lib/waterdrop/instrumentation/callbacks_manager.rb
+- lib/waterdrop/instrumentation/monitor.rb
+- lib/waterdrop/instrumentation/stdout_listener.rb
+- lib/waterdrop/instrumentation/vendors/datadog/dashboard.json
+- lib/waterdrop/instrumentation/vendors/datadog/listener.rb
+- lib/waterdrop/patches/rdkafka/bindings.rb
+- lib/waterdrop/patches/rdkafka/producer.rb
+- lib/waterdrop/producer.rb
+- lib/waterdrop/producer/async.rb
+- lib/waterdrop/producer/buffer.rb
+- lib/waterdrop/producer/builder.rb
+- lib/waterdrop/producer/dummy_client.rb
+- lib/waterdrop/producer/status.rb
+- lib/waterdrop/producer/sync.rb
+- lib/waterdrop/version.rb
 - log/.gitkeep
 - waterdrop.gemspec
-homepage: https://github.com/karafka/waterdrop
+homepage: https://karafka.io
 licenses:
 - MIT
-metadata: {}
+metadata:
+  source_code_uri: https://github.com/karafka/waterdrop
+  rubygems_mfa_required: 'true'
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -181,14 +185,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.6.0
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.25
+rubygems_version: 3.3.4
 signing_key: 
 specification_version: 4
 summary: Kafka messaging made easy!

data/lib/water_drop.rb

@@ -1,36 +0,0 @@
-# frozen_string_literal: true
-
-# External components
-# delegate should be removed because we don't need it, we just add it because of ruby-kafka
-%w[
-  concurrent/array
-  dry-configurable
-  dry/monitor/notifications
-  dry-validation
-  rdkafka
-  json
-  zeitwerk
-  securerandom
-].each { |lib| require lib }
-
-# WaterDrop library
-module WaterDrop
-  class << self
-    # @return [String] root path of this gem
-    def gem_root
-      Pathname.new(File.expand_path('..', __dir__))
-    end
-  end
-end
-
-Zeitwerk::Loader
-  .for_gem
-  .tap { |loader| loader.ignore("#{__dir__}/waterdrop.rb") }
-  .tap(&:setup)
-  .tap(&:eager_load)
-
-# Rdkafka uses a single global callback for things. We bypass that by injecting a manager for
-# each callback type. Callback manager allows us to register more than one callback
-# @note Those managers are also used by Karafka for consumer related statistics
-Rdkafka::Config.statistics_callback = WaterDrop::Instrumentation.statistics_callbacks
-Rdkafka::Config.error_callback = WaterDrop::Instrumentation.error_callbacks