waterdrop 2.1.0 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5a4c3c32e28e4581626a6cf8a10c5dc3aa2dcb72cb3f2517977353c8b6cc4248
-  data.tar.gz: 8d23eb9276f9d5b2138d69542a79403a1a110bd0c02b45e9d11c9e94b7d62fab
+  metadata.gz: db36e5203312e482d0d98a83d3bfb4fd5e274c342e5b3c63829d6bb5f616b52e
+  data.tar.gz: 6e49baf2f5ebd33aded769899fc0eb7f3c130ac97d9a6156fd491d000fd04306
 SHA512:
-  metadata.gz: 384c1d42739f1e01b22439a85c9adb06fdad444f9b709040e9c87e99adb32ce9aee757ea2c8f7b26b2c22e282a36e6c1f1a48df8b471508967d605e36dc646ef
-  data.tar.gz: bfa16b5ff002e36db5e3b45ba823710b5d0c3eda7e32d8f4666b42661cf6de4807d1000887f91d045a890fca2a6afaa68218439c215557430fa992db8123468c
+  metadata.gz: 426d9f910ca5e24a7e86db1ceec672f895186ace830554708e8741d21bb83f929c9278ed5018bff6468b354523ac7cc790cba0b91b8c13dd2bb28e58f8a52dc6
+  data.tar.gz: b3871df6868ce287daf0765f522031cfdd566b96d0e5f8bf3ca1cd9724a7a68d5b3e4e1a33e0bdcc9b65063150d48cfc213e19348d19a6268a309b1c69dcab50

data/.github/workflows/ci.yml

@@ -19,8 +19,6 @@ jobs:
           - '3.1'
           - '3.0'
           - '2.7'
-          - '2.6'
-          - 'jruby-9.3.1.0'
         include:
           - ruby: '3.1'
             coverage: 'true'

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 # WaterDrop changelog
 
+## 2.2.0 (2022-02-18)
+- Add Datadog listener for metrics + errors publishing
+- Add Datadog example dashboard template
+- Update Readme to show Dd instrumentation usage
+- Align the directory namespace convention with gem name (waterdrop => WaterDrop)
+- Introduce a common base for validation contracts
+- Drop CI support for ruby 2.6
+- Require all `kafka` settings to have symbol keys (compatibility with Karafka 2.0 and rdkafka)
+
 ## 2.1.0 (2022-01-03)
 - Ruby 3.1 support
 - Change the error notification key from `error.emitted` to `error.occurred`.

data/Gemfile

@@ -6,8 +6,6 @@ plugin 'diffend'
 
 gemspec
 
-gem 'rdkafka'
-
 group :development do
   gem 'byebug'
 end

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.1.0)
+    waterdrop (2.2.0)
       concurrent-ruby (>= 1.1)
       dry-configurable (~> 0.13)
       dry-monitor (~> 0.5)
@@ -12,17 +12,16 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.1.4.1)
+    activesupport (7.0.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
-      zeitwerk (~> 2.3)
     byebug (11.1.3)
     concurrent-ruby (1.1.9)
-    diff-lcs (1.4.4)
+    diff-lcs (1.5.0)
     docile (1.4.0)
-    dry-configurable (0.13.0)
+    dry-configurable (0.14.0)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.6)
     dry-container (0.9.0)
@@ -34,7 +33,7 @@ GEM
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.5, >= 0.5)
     dry-inflector (0.2.1)
-    dry-initializer (3.0.4)
+    dry-initializer (3.1.1)
     dry-logic (1.2.0)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.5, >= 0.5)
@@ -63,29 +62,29 @@ GEM
       dry-schema (~> 1.8, >= 1.8.0)
     factory_bot (6.2.0)
       activesupport (>= 5.0.0)
-    ffi (1.15.4)
-    i18n (1.8.11)
+    ffi (1.15.5)
+    i18n (1.9.1)
       concurrent-ruby (~> 1.0)
     mini_portile2 (2.7.1)
-    minitest (5.14.4)
+    minitest (5.15.0)
     rake (13.0.6)
     rdkafka (0.11.1)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    rspec (3.10.0)
-      rspec-core (~> 3.10.0)
-      rspec-expectations (~> 3.10.0)
-      rspec-mocks (~> 3.10.0)
-    rspec-core (3.10.1)
-      rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.1)
+    rspec (3.11.0)
+      rspec-core (~> 3.11.0)
+      rspec-expectations (~> 3.11.0)
+      rspec-mocks (~> 3.11.0)
+    rspec-core (3.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-expectations (3.11.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.2)
+      rspec-support (~> 3.11.0)
+    rspec-mocks (3.11.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-support (3.10.3)
+      rspec-support (~> 3.11.0)
+    rspec-support (3.11.0)
     simplecov (0.21.2)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
@@ -94,19 +93,17 @@ GEM
     simplecov_json_formatter (0.1.3)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    zeitwerk (2.5.1)
+    zeitwerk (2.5.4)
 
 PLATFORMS
-  x86_64-darwin
   x86_64-linux
 
 DEPENDENCIES
   byebug
   factory_bot
-  rdkafka
   rspec
   simplecov
   waterdrop!
 
 BUNDLED WITH
-   2.3.4
+   2.3.7

data/README.md

@@ -2,7 +2,7 @@
 
 **Note**: Documentation presented here refers to WaterDrop `2.0.0`.
 
-WaterDrop `2.0` does **not** work with Karafka `1.*` and aims to either work as a standalone producer outside of Karafka `1.*` ecosystem or as a part of not yet released Karafka `2.0.*`.
+WaterDrop `2.0` does **not** work with Karafka `1.*` and aims to either work as a standalone producer outside of Karafka `1.*` ecosystem or as a part of soon to be released Karafka `2.0.*`.
 
 Please refer to [this](https://github.com/karafka/waterdrop/tree/1.4) branch and its documentation for details about WaterDrop `1.*` usage.
 
@@ -36,16 +36,13 @@ It:
 - [Instrumentation](#instrumentation)
   * [Usage statistics](#usage-statistics)
   * [Error notifications](#error-notifications)
+  * [Datadog and StatsD integration](#datadog-and-statsd-integration)
   * [Forking and potential memory problems](#forking-and-potential-memory-problems)
 - [Note on contributions](#note-on-contributions)
 
 ## Installation
 
-```ruby
-gem install waterdrop
-```
-
-or add this to your Gemfile:
+Add this to your Gemfile:
 
 ```ruby
 gem 'waterdrop'
@@ -201,7 +198,7 @@ producer.setup do |config|
   config.kafka = {
     'bootstrap.servers': 'localhost:9092',
     # Accumulate messages for at most 10 seconds
-    'queue.buffering.max.ms' => 10_000
+    'queue.buffering.max.ms': 10_000
   }
 end
 
@@ -288,6 +285,37 @@ producer.close
 
 Note: The metrics returned may not be completely consistent between brokers, toppars and totals, due to the internal asynchronous nature of librdkafka. E.g., the top level tx total may be less than the sum of the broker tx values which it represents.
 
+### Datadog and StatsD integration
+
+WaterDrop comes with (optional) full Datadog and StatsD integration that you can use. To use it:
+
+```ruby
+# require datadog/statsd and the listener as it is not loaded by default
+require 'datadog/statsd'
+require 'waterdrop/instrumentation/vendors/datadog/listener'
+
+# initialize your producer with statistics.interval.ms enabled so the metrics are published
+producer = WaterDrop::Producer.new do |config|
+  config.deliver = true
+  config.kafka = {
+    'bootstrap.servers': 'localhost:9092',
+    'statistics.interval.ms': 1_000
+  }
+end
+
+# initialize the listener with statsd client
+listener = ::WaterDrop::Instrumentation::Vendors::Datadog::Listener.new do |config|
+  config.client = Datadog::Statsd.new('localhost', 8125)
+  # Publish host as a tag alongside the rest of tags
+  config.default_tags = ["host:#{Socket.gethostname}"]
+end
+
+# Subscribe with your listener to your producer and you should be ready to go!
+producer.monitor.subscribe(listener)
+```
+
+You can also find a ready to import DataDog dashboard configuration file that you can use to monitor all of your producer.
+
 ### Error notifications
 
 WaterDrop allows you to listen to all errors that occur while producing messages and in its internal background threads. Things like reconnecting to Kafka upon network errors and others unrelated to publishing messages are all available under `error.occurred` notification key. You can subscribe to this event to ensure your setup is healthy and without any problems that would otherwise go unnoticed as long as messages are delivered.

data/config/errors.yml

@@ -4,3 +4,4 @@ en:
       invalid_key_type: all keys need to be of type String
       invalid_value_type: all values need to be of type String
       max_payload_size: is more than `max_payload_size` config value
+      kafka_key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols

data/lib/{water_drop → waterdrop}/config.rb

@@ -9,7 +9,7 @@ module WaterDrop
 
     # Defaults for kafka settings, that will be overwritten only if not present already
     KAFKA_DEFAULTS = {
-      'client.id' => 'waterdrop'
+      'client.id': 'waterdrop'
     }.freeze
 
     private_constant :KAFKA_DEFAULTS
@@ -63,7 +63,8 @@ module WaterDrop
         yield(config)
 
         merge_kafka_defaults!(config)
-        validate!(config.to_h)
+
+        Contracts::Config.new.validate!(config.to_h, Errors::ConfigurationInvalidError)
 
         ::Rdkafka::Config.logger = config.logger
       end
@@ -82,16 +83,5 @@ module WaterDrop
         config.kafka[key] = value
       end
     end
-
-    # Validates the configuration and if anything is wrong, will raise an exception
-    # @param config_hash [Hash] config hash with setup details
-    # @raise [WaterDrop::Errors::ConfigurationInvalidError] raised when something is wrong with
-    #   the configuration
-    def validate!(config_hash)
-      result = Contracts::Config.new.call(config_hash)
-      return true if result.success?
-
-      raise Errors::ConfigurationInvalidError, result.errors.to_h
-    end
   end
 end

data/lib/waterdrop/contracts/base.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Contracts
+    # Base for all the contracts in WaterDrop
+    class Base < Dry::Validation::Contract
+      config.messages.load_paths << File.join(WaterDrop.gem_root, 'config', 'errors.yml')
+
+      # @param data [Hash] data for validation
+      # @param error_class [Class] error class that should be used when validation fails
+      # @return [Boolean] true
+      # @raise [StandardError] any error provided in the error_class that inherits from the
+      #   standard error
+      def validate!(data, error_class)
+        result = call(data)
+
+        return true if result.success?
+
+        raise error_class, result.errors.to_h
+      end
+    end
+  end
+end

data/lib/{water_drop → waterdrop}/contracts/config.rb

@@ -3,12 +3,7 @@
 module WaterDrop
   module Contracts
     # Contract with validation rules for WaterDrop configuration details
-    class Config < Dry::Validation::Contract
-      # Ensure valid format of each seed broker so that rdkafka doesn't fail silently
-      SEED_BROKER_FORMAT_REGEXP = %r{\A([^:/,]+:[0-9]+)(,[^:/,]+:[0-9]+)*\z}.freeze
-
-      private_constant :SEED_BROKER_FORMAT_REGEXP
-
+    class Config < Base
       params do
         required(:id).filled(:str?)
         required(:logger).filled
@@ -16,9 +11,18 @@ module WaterDrop
         required(:max_payload_size).filled(:int?, gteq?: 1)
         required(:max_wait_timeout).filled(:number?, gteq?: 0)
         required(:wait_timeout).filled(:number?, gt?: 0)
+        required(:kafka).filled(:hash?)
+      end
+
+      # rdkafka allows both symbols and strings as keys for config but then casts them to strings
+      # This can be confusing, so we expect all keys to be symbolized
+      rule(:kafka) do
+        next unless value.is_a?(Hash)
+
+        value.each_key do |key|
+          next if key.is_a?(Symbol)
 
-        required(:kafka).schema do
-          required(:'bootstrap.servers').filled(:str?, format?: SEED_BROKER_FORMAT_REGEXP)
+          key(:"kafka.#{key}").failure(:kafka_key_must_be_a_symbol)
         end
       end
     end

data/lib/{water_drop → waterdrop}/contracts/message.rb

@@ -4,17 +4,15 @@ module WaterDrop
   module Contracts
     # Contract with validation rules for validating that all the message options that
     # we provide to producer ale valid and usable
-    class Message < Dry::Validation::Contract
+    class Message < Base
       # Regex to check that topic has a valid format
-      TOPIC_REGEXP = /\A(\w|-|\.)+\z/.freeze
+      TOPIC_REGEXP = /\A(\w|-|\.)+\z/
 
       # Checks, that the given value is a string
       STRING_ASSERTION = ->(value) { value.is_a?(String) }.to_proc
 
       private_constant :TOPIC_REGEXP, :STRING_ASSERTION
 
-      config.messages.load_paths << File.join(WaterDrop.gem_root, 'config', 'errors.yml')
-
       option :max_payload_size
 
       params do

data/lib/waterdrop/instrumentation/vendors/datadog/dashboard.json

@@ -0,0 +1 @@
+{"title":"WaterDrop producer example dashboard","description":"This dashboard include example setup for monitoring activity of your WaterDrop producer","widgets":[{"id":243951318,"definition":{"title":"Messages produced","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"produced sync","formula":"query1"},{"alias":"produced async","formula":"query2"},{"alias":"flushed sync","formula":"query3"},{"alias":"flushed async","formula":"query4"},{"alias":"acknowledged","formula":"query5"}],"response_format":"timeseries","queries":[{"query":"sum:waterdrop.produced_sync{*}.as_count()","data_source":"metrics","name":"query1"},{"query":"sum:waterdrop.produced_async{*}.as_count()","data_source":"metrics","name":"query2"},{"query":"sum:waterdrop.flushed_sync{*}.as_count()","data_source":"metrics","name":"query3"},{"query":"sum:waterdrop.flushed_async{*}.as_count()","data_source":"metrics","name":"query4"},{"query":"sum:waterdrop.acknowledged{*}.as_count()","data_source":"metrics","name":"query5"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"include_zero":true,"scale":"linear","label":"","min":"auto","max":"auto"}}},{"id":1979626566852990,"definition":{"title":"Messages buffer size","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"max","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"avg:waterdrop.buffer.size.max{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]}},{"id":243951221,"definition":{"title":"Kafka broker API calls","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"API calls","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"sum:waterdrop.calls{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"include_zero":true,"scale":"linear","label":"","min":"auto","max":"auto"}}},{"id":243951952,"definition":{"title":"Producer queue size","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Queue size average","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"max:waterdrop.queue.size.avg{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"},{"formulas":[{"alias":"Queue size max","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"max:waterdrop.queue.size.max{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"include_zero":true,"scale":"linear","label":"","min":"auto","max":"auto"}}},{"id":243951263,"definition":{"title":"Producer queue latency","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Average latency","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"avg:waterdrop.queue.latency.avg{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"},{"formulas":[{"alias":"Latency p95","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"avg:waterdrop.queue.latency.p95{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"},{"formulas":[{"alias":"Latency p99","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"avg:waterdrop.queue.latency.p99{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"include_zero":true,"scale":"linear","label":"","min":"auto","max":"auto"}}},{"id":243951276,"definition":{"title":"Producer network latency","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Average latency","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"avg:waterdrop.request_size.avg{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"},{"formulas":[{"alias":"Latency p95","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"avg:waterdrop.network.latency.p95{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"},{"formulas":[{"alias":"Latency p99","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"avg:waterdrop.network.latency.p99{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"include_zero":true,"scale":"linear","label":"","min":"auto","max":"auto"}}},{"id":243954928,"definition":{"title":"Producer errors","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"response_format":"timeseries","queries":[{"query":"sum:waterdrop.error_occurred{*}.as_count()","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"include_zero":true,"scale":"linear","label":"","min":"auto","max":"auto"}}}],"template_variables":[],"layout_type":"ordered","is_read_only":false,"notify_list":[],"reflow_type":"auto","id":"rnr-kgh-dna"}

data/lib/waterdrop/instrumentation/vendors/datadog/listener.rb

@@ -0,0 +1,197 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Instrumentation
+    # Namespace for vendor specific instrumentation
+    module Vendors
+      # Datadog specific instrumentation
+      module Datadog
+        # Listener that can be used to subscribe to WaterDrop producer to receive stats via StatsD
+        # and/or Datadog
+        #
+        # @note You need to setup the `dogstatsd-ruby` client and assign it
+        class Listener
+          include Dry::Configurable
+
+          # Value object for storing a single rdkafka metric publishing details
+          RdKafkaMetric = Struct.new(:type, :scope, :name, :key_location)
+
+          # Namespace under which the DD metrics should be published
+          setting :namespace, default: 'waterdrop', reader: true
+
+          # Datadog client that we should use to publish the metrics
+          setting :client, reader: true
+
+          # Default tags we want to publish (for example hostname)
+          # Format as followed (example for hostname): `["host:#{Socket.gethostname}"]`
+          setting :default_tags, default: [], reader: true
+
+          # All the rdkafka metrics we want to publish
+          #
+          # By default we publish quite a lot so this can be tuned
+          # Note, that the once with `_d` come from WaterDrop, not rdkafka or Kafka
+          setting :rd_kafka_metrics, reader: true, default: [
+            # Client metrics
+            RdKafkaMetric.new(:count, :root, 'calls', 'tx_d'),
+            RdKafkaMetric.new(:histogram, :root, 'queue.size', 'msg_cnt_d'),
+
+            # Broker metrics
+            RdKafkaMetric.new(:count, :brokers, 'deliver.attempts', 'txretries_d'),
+            RdKafkaMetric.new(:count, :brokers, 'deliver.errors', 'txerrs_d'),
+            RdKafkaMetric.new(:count, :brokers, 'receive.errors', 'rxerrs_d'),
+            RdKafkaMetric.new(:gauge, :brokers, 'queue.latency.avg', %w[outbuf_latency avg]),
+            RdKafkaMetric.new(:gauge, :brokers, 'queue.latency.p95', %w[outbuf_latency p95]),
+            RdKafkaMetric.new(:gauge, :brokers, 'queue.latency.p99', %w[outbuf_latency p99]),
+            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.avg', %w[rtt avg]),
+            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.p95', %w[rtt p95]),
+            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.p99', %w[rtt p99])
+          ].freeze
+
+          # @param block [Proc] configuration block
+          def initialize(&block)
+            setup(&block) if block
+          end
+
+          # @param block [Proc] configuration block
+          # @note We define this alias to be consistent with `WaterDrop#setup`
+          def setup(&block)
+            configure(&block)
+          end
+
+          # Hooks up to WaterDrop instrumentation for emitted statistics
+          #
+          # @param event [Dry::Events::Event]
+          def on_statistics_emitted(event)
+            statistics = event[:statistics]
+
+            rd_kafka_metrics.each do |metric|
+              report_metric(metric, statistics)
+            end
+          end
+
+          # Increases the errors count by 1
+          #
+          # @param _event [Dry::Events::Event]
+          def on_error_occurred(_event)
+            client.count(
+              namespaced_metric('error_occurred'),
+              1,
+              tags: default_tags
+            )
+          end
+
+          # Increases acknowledged messages counter
+          # @param _event [Dry::Events::Event]
+          def on_message_acknowledged(_event)
+            client.increment(
+              namespaced_metric('acknowledged'),
+              tags: default_tags
+            )
+          end
+
+          %i[
+            produced_sync
+            produced_async
+          ].each do |event_scope|
+            class_eval <<~METHODS, __FILE__, __LINE__ + 1
+              # @param event [Dry::Events::Event]
+              def on_message_#{event_scope}(event)
+                report_message(event[:message][:topic], :#{event_scope})
+              end
+
+              # @param event [Dry::Events::Event]
+              def on_messages_#{event_scope}(event)
+                event[:messages].each do |message|
+                  report_message(message[:topic], :#{event_scope})
+                end
+              end
+            METHODS
+          end
+
+          # Reports the buffer usage when anything is added to the buffer
+          %i[
+            message_buffered
+            messages_buffered
+          ].each do |event_scope|
+            class_eval <<~METHODS, __FILE__, __LINE__ + 1
+              # @param event [Dry::Events::Event]
+              def on_#{event_scope}(event)
+                client.histogram(
+                  namespaced_metric('buffer.size'),
+                  event[:buffer].size,
+                  tags: default_tags
+                )
+              end
+            METHODS
+          end
+
+          # Events that support many messages only
+          # Reports data flushing operation (production from the buffer)
+          %i[
+            flushed_sync
+            flushed_async
+          ].each do |event_scope|
+            class_eval <<~METHODS, __FILE__, __LINE__ + 1
+              # @param event [Dry::Events::Event]
+              def on_buffer_#{event_scope}(event)
+                event[:messages].each do |message|
+                  report_message(message[:topic], :#{event_scope})
+                end
+              end
+            METHODS
+          end
+
+          private
+
+          # Report that a message has been produced to a topic.
+          # @param topic [String] Kafka topic
+          # @param method_name [Symbol] method from which this message operation comes
+          def report_message(topic, method_name)
+            client.increment(
+              namespaced_metric(method_name),
+              tags: default_tags + ["topic:#{topic}"]
+            )
+          end
+
+          # Wraps metric name in listener's namespace
+          # @param metric_name [String] RdKafkaMetric name
+          # @return [String]
+          def namespaced_metric(metric_name)
+            "#{namespace}.#{metric_name}"
+          end
+
+          # Reports a given metric statistics to Datadog
+          # @param metric [RdKafkaMetric] metric value object
+          # @param statistics [Hash] hash with all the statistics emitted
+          def report_metric(metric, statistics)
+            case metric.scope
+            when :root
+              client.public_send(
+                metric.type,
+                namespaced_metric(metric.name),
+                statistics.fetch(*metric.key_location),
+                tags: default_tags
+              )
+            when :brokers
+              statistics.fetch('brokers').each_value do |broker_statistics|
+                # Skip bootstrap nodes
+                # Bootstrap nodes have nodeid -1, other nodes have positive
+                # node ids
+                next if broker_statistics['nodeid'] == -1
+
+                client.public_send(
+                  metric.type,
+                  namespaced_metric(metric.name),
+                  broker_statistics.dig(*metric.key_location),
+                  tags: default_tags + ["broker:#{broker_statistics['nodename']}"]
+                )
+              end
+            else
+              raise ArgumentError, metric.scope
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/{water_drop → waterdrop}/producer/buffer.rb

@@ -24,7 +24,8 @@ module WaterDrop
         @monitor.instrument(
           'message.buffered',
           producer_id: id,
-          message: message
+          message: message,
+          buffer: @messages
         ) { @messages << message }
       end
 
@@ -41,7 +42,8 @@ module WaterDrop
         @monitor.instrument(
           'messages.buffered',
           producer_id: id,
-          messages: messages
+          messages: messages,
+          buffer: @messages
         ) do
           messages.each { |message| @messages << message }
           messages

data/lib/{water_drop → waterdrop}/producer.rb

@@ -150,13 +150,7 @@ module WaterDrop
     # @param message [Hash] message we want to send
     # @raise [Karafka::Errors::MessageInvalidError]
     def validate_message!(message)
-      result = @contract.call(message)
-      return if result.success?
-
-      raise Errors::MessageInvalidError, [
-        result.errors.to_h,
-        message
-      ]
+      @contract.validate!(message, Errors::MessageInvalidError)
     end
   end
 end

data/lib/{water_drop → waterdrop}/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '2.1.0'
+  VERSION = '2.2.0'
 end

data/lib/waterdrop.rb

@@ -1,4 +1,37 @@
 # frozen_string_literal: true
 
-# This file is used as a compatibility step
-require 'water_drop'
+# External components
+# delegate should be removed because we don't need it, we just add it because of ruby-kafka
+%w[
+  concurrent/array
+  dry-configurable
+  dry/monitor/notifications
+  dry-validation
+  rdkafka
+  json
+  zeitwerk
+  securerandom
+].each { |lib| require lib }
+
+# WaterDrop library
+module WaterDrop
+  class << self
+    # @return [String] root path of this gem
+    def gem_root
+      Pathname.new(File.expand_path('..', __dir__))
+    end
+  end
+end
+
+loader = Zeitwerk::Loader.for_gem
+loader.inflector.inflect('waterdrop' => 'WaterDrop')
+# Do not load vendors instrumentation components. Those need to be required manually if needed
+loader.ignore("#{__dir__}/waterdrop/instrumentation/vendors/**/*.rb")
+loader.setup
+loader.eager_load
+
+# Rdkafka uses a single global callback for things. We bypass that by injecting a manager for
+# each callback type. Callback manager allows us to register more than one callback
+# @note Those managers are also used by Karafka for consumer related statistics
+Rdkafka::Config.statistics_callback = WaterDrop::Instrumentation.statistics_callbacks
+Rdkafka::Config.error_callback = WaterDrop::Instrumentation.error_callbacks

data/waterdrop.gemspec

@@ -3,7 +3,7 @@
 lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
-require 'water_drop/version'
+require 'waterdrop/version'
 
 Gem::Specification.new do |spec|
   spec.name          = 'waterdrop'
@@ -23,7 +23,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'rdkafka', '>= 0.10'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
-  spec.required_ruby_version = '>= 2.6.0'
+  spec.required_ruby_version = '>= 2.7'
 
   if $PROGRAM_NAME.end_with?('gem')
     spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
@@ -33,5 +33,9 @@ Gem::Specification.new do |spec|
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = %w[lib]
-  spec.metadata      = { 'source_code_uri' => 'https://github.com/karafka/waterdrop' }
+
+  spec.metadata = {
+    'source_code_uri' => 'https://github.com/karafka/waterdrop',
+    'rubygems_mfa_required' => 'true'
+  }
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.2.0
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -34,7 +34,7 @@ cert_chain:
   R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
   pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
   -----END CERTIFICATE-----
-date: 2022-01-03 00:00:00.000000000 Z
+date: 2022-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -142,31 +142,33 @@ files:
 - certs/mensfeld.pem
 - config/errors.yml
 - docker-compose.yml
-- lib/water_drop.rb
-- lib/water_drop/config.rb
-- lib/water_drop/contracts.rb
-- lib/water_drop/contracts/config.rb
-- lib/water_drop/contracts/message.rb
-- lib/water_drop/errors.rb
-- lib/water_drop/instrumentation.rb
-- lib/water_drop/instrumentation/callbacks/delivery.rb
-- lib/water_drop/instrumentation/callbacks/error.rb
-- lib/water_drop/instrumentation/callbacks/statistics.rb
-- lib/water_drop/instrumentation/callbacks/statistics_decorator.rb
-- lib/water_drop/instrumentation/callbacks_manager.rb
-- lib/water_drop/instrumentation/monitor.rb
-- lib/water_drop/instrumentation/stdout_listener.rb
-- lib/water_drop/patches/rdkafka/bindings.rb
-- lib/water_drop/patches/rdkafka/producer.rb
-- lib/water_drop/producer.rb
-- lib/water_drop/producer/async.rb
-- lib/water_drop/producer/buffer.rb
-- lib/water_drop/producer/builder.rb
-- lib/water_drop/producer/dummy_client.rb
-- lib/water_drop/producer/status.rb
-- lib/water_drop/producer/sync.rb
-- lib/water_drop/version.rb
 - lib/waterdrop.rb
+- lib/waterdrop/config.rb
+- lib/waterdrop/contracts.rb
+- lib/waterdrop/contracts/base.rb
+- lib/waterdrop/contracts/config.rb
+- lib/waterdrop/contracts/message.rb
+- lib/waterdrop/errors.rb
+- lib/waterdrop/instrumentation.rb
+- lib/waterdrop/instrumentation/callbacks/delivery.rb
+- lib/waterdrop/instrumentation/callbacks/error.rb
+- lib/waterdrop/instrumentation/callbacks/statistics.rb
+- lib/waterdrop/instrumentation/callbacks/statistics_decorator.rb
+- lib/waterdrop/instrumentation/callbacks_manager.rb
+- lib/waterdrop/instrumentation/monitor.rb
+- lib/waterdrop/instrumentation/stdout_listener.rb
+- lib/waterdrop/instrumentation/vendors/datadog/dashboard.json
+- lib/waterdrop/instrumentation/vendors/datadog/listener.rb
+- lib/waterdrop/patches/rdkafka/bindings.rb
+- lib/waterdrop/patches/rdkafka/producer.rb
+- lib/waterdrop/producer.rb
+- lib/waterdrop/producer/async.rb
+- lib/waterdrop/producer/buffer.rb
+- lib/waterdrop/producer/builder.rb
+- lib/waterdrop/producer/dummy_client.rb
+- lib/waterdrop/producer/status.rb
+- lib/waterdrop/producer/sync.rb
+- lib/waterdrop/version.rb
 - log/.gitkeep
 - waterdrop.gemspec
 homepage: https://karafka.io
@@ -174,6 +176,7 @@ licenses:
 - MIT
 metadata:
   source_code_uri: https://github.com/karafka/waterdrop
+  rubygems_mfa_required: 'true'
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -182,14 +185,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.6.0
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.3
+rubygems_version: 3.3.4
 signing_key: 
 specification_version: 4
 summary: Kafka messaging made easy!

data/lib/water_drop.rb

@@ -1,36 +0,0 @@
-# frozen_string_literal: true
-
-# External components
-# delegate should be removed because we don't need it, we just add it because of ruby-kafka
-%w[
-  concurrent/array
-  dry-configurable
-  dry/monitor/notifications
-  dry-validation
-  rdkafka
-  json
-  zeitwerk
-  securerandom
-].each { |lib| require lib }
-
-# WaterDrop library
-module WaterDrop
-  class << self
-    # @return [String] root path of this gem
-    def gem_root
-      Pathname.new(File.expand_path('..', __dir__))
-    end
-  end
-end
-
-Zeitwerk::Loader
-  .for_gem
-  .tap { |loader| loader.ignore("#{__dir__}/waterdrop.rb") }
-  .tap(&:setup)
-  .tap(&:eager_load)
-
-# Rdkafka uses a single global callback for things. We bypass that by injecting a manager for
-# each callback type. Callback manager allows us to register more than one callback
-# @note Those managers are also used by Karafka for consumer related statistics
-Rdkafka::Config.statistics_callback = WaterDrop::Instrumentation.statistics_callbacks
-Rdkafka::Config.error_callback = WaterDrop::Instrumentation.error_callbacks