waterdrop 2.0.0 → 2.4.0

data/lib/waterdrop/instrumentation/callbacks/error.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Instrumentation
+    module Callbacks
+      # Callback that kicks in when error occurs and is published in a background thread
+      class Error
+        # @param producer_id [String] id of the current producer
+        # @param client_name [String] rdkafka client name
+        # @param monitor [WaterDrop::Instrumentation::Monitor] monitor we are using
+        def initialize(producer_id, client_name, monitor)
+          @producer_id = producer_id
+          @client_name = client_name
+          @monitor = monitor
+        end
+
+        # Runs the instrumentation monitor with error
+        # @param client_name [String] rdkafka client name
+        # @param error [Rdkafka::Error] error that occurred
+        # @note It will only instrument on errors of the client of our producer
+        def call(client_name, error)
+          # Emit only errors related to our client
+          # Same as with statistics (mor explanation there)
+          return unless @client_name == client_name
+
+          @monitor.instrument(
+            'error.occurred',
+            error: error,
+            producer_id: @producer_id,
+            type: 'librdkafka.error'
+          )
+        end
+      end
+    end
+  end
+end

data/lib/waterdrop/instrumentation/callbacks/statistics.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Instrumentation
+    # Namespace for handlers of callbacks emitted by the kafka client lib
+    module Callbacks
+      # Statistics callback handler
+      # @note We decorate the statistics with our own decorator because some of the metrics from
+      #   rdkafka are absolute. For example number of sent messages increases not in reference to
+      #   previous statistics emit but from the beginning of the process. We decorate it with diff
+      #   of all the numeric values against the data from the previous callback emit
+      class Statistics
+        # @param producer_id [String] id of the current producer
+        # @param client_name [String] rdkafka client name
+        # @param monitor [WaterDrop::Instrumentation::Monitor] monitor we are using
+        def initialize(producer_id, client_name, monitor)
+          @producer_id = producer_id
+          @client_name = client_name
+          @monitor = monitor
+          @statistics_decorator = StatisticsDecorator.new
+        end
+
+        # Emits decorated statistics to the monitor
+        # @param statistics [Hash] rdkafka statistics
+        def call(statistics)
+          # Emit only statistics related to our client
+          # rdkafka does not have per-instance statistics hook, thus we need to make sure that we
+          # emit only stats that are related to current producer. Otherwise we would emit all of
+          # all the time.
+          return unless @client_name == statistics['name']
+
+          @monitor.instrument(
+            'statistics.emitted',
+            producer_id: @producer_id,
+            statistics: @statistics_decorator.call(statistics)
+          )
+        end
+      end
+    end
+  end
+end

data/lib/waterdrop/instrumentation/callbacks/statistics_decorator.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Instrumentation
+    module Callbacks
+      # Many of the librdkafka statistics are absolute values instead of a gauge.
+      # This means, that for example number of messages sent is an absolute growing value
+      # instead of being a value of messages sent from the last statistics report.
+      # This decorator calculates the diff against previously emited stats, so we get also
+      # the diff together with the original values
+      class StatisticsDecorator
+        def initialize
+          @previous = {}.freeze
+        end
+
+        # @param emited_stats [Hash] original emited statistics
+        # @return [Hash] emited statistics extended with the diff data
+        # @note We modify the emited statistics, instead of creating new. Since we don't expose
+        #   any API to get raw data, users can just assume that the result of this decoration is
+        #   the proper raw stats that they can use
+        def call(emited_stats)
+          diff(
+            @previous,
+            emited_stats
+          )
+
+          @previous = emited_stats
+
+          emited_stats.freeze
+        end
+
+        private
+
+        # Calculates the diff of the provided values and modifies in place the emited statistics
+        #
+        # @param previous [Object] previous value from the given scope in which
+        #   we are
+        # @param current [Object] current scope from emitted statistics
+        # @return [Object] the diff if the values were numerics or the current scope
+        def diff(previous, current)
+          if current.is_a?(Hash)
+            # @note We cannot use #each_key as we modify the content of the current scope
+            #   in place (in case it's a hash)
+            current.keys.each do |key|
+              append(
+                current,
+                key,
+                diff((previous || {})[key], (current || {})[key])
+              )
+            end
+          end
+
+          # Diff can be computed only for numerics
+          return current unless current.is_a?(Numeric)
+          # If there was no previous value, delta is always zero
+          return 0 unless previous
+          # Should never happen but just in case, a type changed in between stats
+          return current unless previous.is_a?(Numeric)
+
+          current - previous
+        end
+
+        # Appends the result of the diff to a given key as long as the result is numeric
+        #
+        # @param current [Hash] current scope
+        # @param key [Symbol] key based on which we were diffing
+        # @param result [Object] diff result
+        def append(current, key, result)
+          return unless result.is_a?(Numeric)
+          return if current.frozen?
+
+          current["#{key}_d"] = result
+        end
+      end
+    end
+  end
+end

data/lib/waterdrop/instrumentation/callbacks_manager.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Instrumentation
+    # This manager allows us to register multiple callbacks into a hook that is suppose to support
+    # a single callback
+    class CallbacksManager
+      # @return [::WaterDrop::Instrumentation::CallbacksManager]
+      def initialize
+        @callbacks = Concurrent::Hash.new
+      end
+
+      # Invokes all the callbacks registered one after another
+      #
+      # @param args [Object] any args that should go to the callbacks
+      # @note We do not use `#each_value` here on purpose. With it being used, we cannot dispatch
+      #   callbacks and add new at the same time. Since we don't know when and in what thread
+      #   things are going to be added to the manager, we need to extract values into an array and
+      #   run it. That way we can add new things the same time.
+      def call(*args)
+        @callbacks.values.each { |callback| callback.call(*args) }
+      end
+
+      # Adds a callback to the manager
+      #
+      # @param id [String] id of the callback (used when deleting it)
+      # @param callable [#call] object that responds to a `#call` method
+      def add(id, callable)
+        @callbacks[id] = callable
+      end
+
+      # Removes the callback from the manager
+      # @param id [String] id of the callback we want to remove
+      def delete(id)
+        @callbacks.delete(id)
+      end
+    end
+  end
+end

data/lib/{water_drop/instrumentation/stdout_listener.rb → waterdrop/instrumentation/logger_listener.rb}

@@ -6,8 +6,8 @@ module WaterDrop
     # It can be removed/replaced or anything without any harm to the Waterdrop flow
     # @note It is a module as we can use it then as a part of the Karafka framework listener
     #   as well as we can use it standalone
-    class StdoutListener
-      # @param logger [Object] stdout logger we want to use
+    class LoggerListener
+      # @param logger [Object] logger we want to use
       def initialize(logger)
         @logger = logger
       end
@@ -51,7 +51,7 @@ module WaterDrop
         message = event[:message]
 
         info(event, "Buffering of a message to '#{message[:topic]}' topic")
-        debug(event, [message, event[:producer].messages.size])
+        debug(event, [message])
       end
 
       # @param event [Dry::Events::Event] event that happened with the details
@@ -59,7 +59,7 @@ module WaterDrop
         messages = event[:messages]
 
         info(event, "Buffering of #{messages.size} messages")
-        debug(event, [messages, event[:producer].messages.size])
+        debug(event, [messages, messages.size])
       end
 
       # @param event [Dry::Events::Event] event that happened with the details
@@ -70,15 +70,6 @@ module WaterDrop
         debug(event, messages)
       end
 
-      # @param event [Dry::Events::Event] event that happened with the details
-      def on_buffer_flushed_async_error(event)
-        messages = event[:messages]
-        error = event[:error]
-
-        error(event, "Async flushing of #{messages.size} failed due to: #{error}")
-        debug(event, messages)
-      end
-
       # @param event [Dry::Events::Event] event that happened with the details
       def on_buffer_flushed_sync(event)
         messages = event[:messages]
@@ -87,19 +78,19 @@ module WaterDrop
         debug(event, messages)
       end
 
-      # @param event [Dry::Events::Event] event that happened with the details
-      def on_buffer_flushed_sync_error(event)
-        messages = event[:dispatched]
-        error = event[:error]
-
-        error(event, "Sync flushing of #{messages.size} failed due to: #{error}")
-        debug(event, messages)
-      end
-
       # @param event [Dry::Events::Event] event that happened with the details
       def on_producer_closed(event)
         info event, 'Closing producer'
-        debug event, event[:producer].messages.size
+        debug event, ''
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the error details
+      def on_error_occurred(event)
+        error = event[:error]
+        type = event[:type]
+
+        error(event, "Error occurred: #{error} - #{type}")
+        debug(event, '')
       end
 
       private
@@ -107,19 +98,19 @@ module WaterDrop
       # @param event [Dry::Events::Event] event that happened with the details
       # @param log_message [String] message we want to publish
       def debug(event, log_message)
-        @logger.debug("[#{event[:producer].id}] #{log_message}")
+        @logger.debug("[#{event[:producer_id]}] #{log_message}")
       end
 
       # @param event [Dry::Events::Event] event that happened with the details
       # @param log_message [String] message we want to publish
       def info(event, log_message)
-        @logger.info("[#{event[:producer].id}] #{log_message} took #{event[:time]} ms")
+        @logger.info("[#{event[:producer_id]}] #{log_message} took #{event[:time]} ms")
       end
 
       # @param event [Dry::Events::Event] event that happened with the details
       # @param log_message [String] message we want to publish
       def error(event, log_message)
-        @logger.error("[#{event[:producer].id}] #{log_message}")
+        @logger.error("[#{event[:producer_id]}] #{log_message}")
       end
     end
   end

data/lib/waterdrop/instrumentation/monitor.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Instrumentation
+    # WaterDrop instrumentation monitor that we use to publish events
+    # By default uses our internal notifications bus but can be used with
+    # `ActiveSupport::Notifications` as well
+    class Monitor < ::Karafka::Core::Monitoring::Monitor
+      # @param notifications_bus [Object] either our internal notifications bus or
+      #   `ActiveSupport::Notifications`
+      # @param namespace [String, nil] namespace for events or nil if no namespace
+      def initialize(
+        notifications_bus = WaterDrop::Instrumentation::Notifications.new,
+        namespace = nil
+      )
+        super(notifications_bus, namespace)
+      end
+    end
+  end
+end

data/lib/{water_drop/instrumentation/monitor.rb → waterdrop/instrumentation/notifications.rb}

@@ -2,37 +2,36 @@
 
 module WaterDrop
   module Instrumentation
-    # Monitor is used to hookup external monitoring services to monitor how WaterDrop works
-    # Since it is a pub-sub based on dry-monitor, you can use as many subscribers/loggers at the
-    # same time, which means that you might have for example file logging and NewRelic at the same
-    # time
-    # @note This class acts as a singleton because we are only permitted to have single monitor
-    #   per running process (just as logger)
-    class Monitor < Dry::Monitor::Notifications
+    # Instrumented is used to hookup external monitoring services to monitor how WaterDrop works
+    class Notifications < ::Karafka::Core::Monitoring::Notifications
       # List of events that we support in the system and to which a monitor client can hook up
       # @note The non-error once support timestamp benchmarking
       EVENTS = %w[
         producer.closed
+
         message.produced_async
         message.produced_sync
+        message.acknowledged
+        message.buffered
+
         messages.produced_async
         messages.produced_sync
-        message.buffered
         messages.buffered
-        message.acknowledged
+
         buffer.flushed_async
-        buffer.flushed_async.error
         buffer.flushed_sync
-        buffer.flushed_sync.error
+
         statistics.emitted
+
+        error.occurred
       ].freeze
 
       private_constant :EVENTS
 
       # @return [WaterDrop::Instrumentation::Monitor] monitor instance for system instrumentation
       def initialize
-        super(:waterdrop)
-        EVENTS.each(&method(:register_event))
+        super
+        EVENTS.each { |event| register_event(event) }
       end
     end
   end

data/lib/waterdrop/instrumentation/vendors/datadog/dashboard.json

@@ -0,0 +1 @@
+{"title":"WaterDrop producer example dashboard","description":"This dashboard include example setup for monitoring activity of your WaterDrop producer","widgets":[{"id":243951318,"definition":{"title":"Messages produced","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"produced sync","formula":"query1"},{"alias":"produced async","formula":"query2"},{"alias":"flushed sync","formula":"query3"},{"alias":"flushed async","formula":"query4"},{"alias":"acknowledged","formula":"query5"}],"response_format":"timeseries","queries":[{"query":"sum:waterdrop.produced_sync{*}.as_count()","data_source":"metrics","name":"query1"},{"query":"sum:waterdrop.produced_async{*}.as_count()","data_source":"metrics","name":"query2"},{"query":"sum:waterdrop.flushed_sync{*}.as_count()","data_source":"metrics","name":"query3"},{"query":"sum:waterdrop.flushed_async{*}.as_count()","data_source":"metrics","name":"query4"},{"query":"sum:waterdrop.acknowledged{*}.as_count()","data_source":"metrics","name":"query5"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"include_zero":true,"scale":"linear","label":"","min":"auto","max":"auto"}}},{"id":1979626566852990,"definition":{"title":"Messages buffer size","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"max","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"avg:waterdrop.buffer.size.max{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]}},{"id":243951221,"definition":{"title":"Kafka broker API calls","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"API calls","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"sum:waterdrop.calls{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"include_zero":true,"scale":"linear","label":"","min":"auto","max":"auto"}}},{"id":243951952,"definition":{"title":"Producer queue size","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Queue size average","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"max:waterdrop.queue.size.avg{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"},{"formulas":[{"alias":"Queue size max","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"max:waterdrop.queue.size.max{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"include_zero":true,"scale":"linear","label":"","min":"auto","max":"auto"}}},{"id":243951263,"definition":{"title":"Producer queue latency","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Average latency","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"avg:waterdrop.queue.latency.avg{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"},{"formulas":[{"alias":"Latency p95","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"avg:waterdrop.queue.latency.p95{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"},{"formulas":[{"alias":"Latency p99","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"avg:waterdrop.queue.latency.p99{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"include_zero":true,"scale":"linear","label":"","min":"auto","max":"auto"}}},{"id":243951276,"definition":{"title":"Producer network latency","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Average latency","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"avg:waterdrop.request_size.avg{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"},{"formulas":[{"alias":"Latency p95","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"avg:waterdrop.network.latency.p95{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"},{"formulas":[{"alias":"Latency p99","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"avg:waterdrop.network.latency.p99{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"include_zero":true,"scale":"linear","label":"","min":"auto","max":"auto"}}},{"id":243954928,"definition":{"title":"Producer errors","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"response_format":"timeseries","queries":[{"query":"sum:waterdrop.error_occurred{*}.as_count()","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"include_zero":true,"scale":"linear","label":"","min":"auto","max":"auto"}}}],"template_variables":[],"layout_type":"ordered","is_read_only":false,"notify_list":[],"reflow_type":"auto","id":"rnr-kgh-dna"}

data/lib/waterdrop/instrumentation/vendors/datadog/listener.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Instrumentation
+    # Namespace for vendor specific instrumentation
+    module Vendors
+      # Datadog specific instrumentation
+      module Datadog
+        # Listener that can be used to subscribe to WaterDrop producer to receive stats via StatsD
+        # and/or Datadog
+        #
+        # @note You need to setup the `dogstatsd-ruby` client and assign it
+        class Listener
+          include ::Karafka::Core::Configurable
+          extend Forwardable
+
+          def_delegators :config, :client, :rd_kafka_metrics, :namespace, :default_tags
+
+          # Value object for storing a single rdkafka metric publishing details
+          RdKafkaMetric = Struct.new(:type, :scope, :name, :key_location)
+
+          # Namespace under which the DD metrics should be published
+          setting :namespace, default: 'waterdrop'
+
+          # Datadog client that we should use to publish the metrics
+          setting :client
+
+          # Default tags we want to publish (for example hostname)
+          # Format as followed (example for hostname): `["host:#{Socket.gethostname}"]`
+          setting :default_tags, default: []
+
+          # All the rdkafka metrics we want to publish
+          #
+          # By default we publish quite a lot so this can be tuned
+          # Note, that the once with `_d` come from WaterDrop, not rdkafka or Kafka
+          setting :rd_kafka_metrics, default: [
+            # Client metrics
+            RdKafkaMetric.new(:count, :root, 'calls', 'tx_d'),
+            RdKafkaMetric.new(:histogram, :root, 'queue.size', 'msg_cnt_d'),
+
+            # Broker metrics
+            RdKafkaMetric.new(:count, :brokers, 'deliver.attempts', 'txretries_d'),
+            RdKafkaMetric.new(:count, :brokers, 'deliver.errors', 'txerrs_d'),
+            RdKafkaMetric.new(:count, :brokers, 'receive.errors', 'rxerrs_d'),
+            RdKafkaMetric.new(:gauge, :brokers, 'queue.latency.avg', %w[outbuf_latency avg]),
+            RdKafkaMetric.new(:gauge, :brokers, 'queue.latency.p95', %w[outbuf_latency p95]),
+            RdKafkaMetric.new(:gauge, :brokers, 'queue.latency.p99', %w[outbuf_latency p99]),
+            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.avg', %w[rtt avg]),
+            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.p95', %w[rtt p95]),
+            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.p99', %w[rtt p99])
+          ].freeze
+
+          configure
+
+          # @param block [Proc] configuration block
+          def initialize(&block)
+            configure
+            setup(&block) if block
+          end
+
+          # @param block [Proc] configuration block
+          # @note We define this alias to be consistent with `WaterDrop#setup`
+          def setup(&block)
+            configure(&block)
+          end
+
+          # Hooks up to WaterDrop instrumentation for emitted statistics
+          #
+          # @param event [WaterDrop::Monitor::Event]
+          def on_statistics_emitted(event)
+            statistics = event[:statistics]
+
+            rd_kafka_metrics.each do |metric|
+              report_metric(metric, statistics)
+            end
+          end
+
+          # Increases the errors count by 1
+          #
+          # @param _event [WaterDrop::Monitor::Event]
+          def on_error_occurred(_event)
+            count('error_occurred', 1, tags: default_tags)
+          end
+
+          # Increases acknowledged messages counter
+          # @param _event [WaterDrop::Monitor::Event]
+          def on_message_acknowledged(_event)
+            increment('acknowledged', tags: default_tags)
+          end
+
+          %i[
+            produced_sync
+            produced_async
+          ].each do |event_scope|
+            class_eval <<~METHODS, __FILE__, __LINE__ + 1
+              # @param event [WaterDrop::Monitor::Event]
+              def on_message_#{event_scope}(event)
+                report_message(event[:message][:topic], :#{event_scope})
+              end
+
+              # @param event [WaterDrop::Monitor::Event]
+              def on_messages_#{event_scope}(event)
+                event[:messages].each do |message|
+                  report_message(message[:topic], :#{event_scope})
+                end
+              end
+            METHODS
+          end
+
+          # Reports the buffer usage when anything is added to the buffer
+          %i[
+            message_buffered
+            messages_buffered
+          ].each do |event_scope|
+            class_eval <<~METHODS, __FILE__, __LINE__ + 1
+              # @param event [WaterDrop::Monitor::Event]
+              def on_#{event_scope}(event)
+                histogram(
+                  'buffer.size',
+                  event[:buffer].size,
+                  tags: default_tags
+                )
+              end
+            METHODS
+          end
+
+          # Events that support many messages only
+          # Reports data flushing operation (production from the buffer)
+          %i[
+            flushed_sync
+            flushed_async
+          ].each do |event_scope|
+            class_eval <<~METHODS, __FILE__, __LINE__ + 1
+              # @param event [WaterDrop::Monitor::Event]
+              def on_buffer_#{event_scope}(event)
+                event[:messages].each do |message|
+                  report_message(message[:topic], :#{event_scope})
+                end
+              end
+            METHODS
+          end
+
+          private
+
+          %i[
+            count
+            gauge
+            histogram
+            increment
+            decrement
+          ].each do |metric_type|
+            class_eval <<~METHODS, __FILE__, __LINE__ + 1
+              def #{metric_type}(key, *args)
+                client.#{metric_type}(
+                  namespaced_metric(key),
+                  *args
+                )
+              end
+            METHODS
+          end
+
+          # Report that a message has been produced to a topic.
+          # @param topic [String] Kafka topic
+          # @param method_name [Symbol] method from which this message operation comes
+          def report_message(topic, method_name)
+            increment(method_name, tags: default_tags + ["topic:#{topic}"])
+          end
+
+          # Wraps metric name in listener's namespace
+          # @param metric_name [String] RdKafkaMetric name
+          # @return [String]
+          def namespaced_metric(metric_name)
+            "#{namespace}.#{metric_name}"
+          end
+
+          # Reports a given metric statistics to Datadog
+          # @param metric [RdKafkaMetric] metric value object
+          # @param statistics [Hash] hash with all the statistics emitted
+          def report_metric(metric, statistics)
+            case metric.scope
+            when :root
+              public_send(
+                metric.type,
+                metric.name,
+                statistics.fetch(*metric.key_location),
+                tags: default_tags
+              )
+            when :brokers
+              statistics.fetch('brokers').each_value do |broker_statistics|
+                # Skip bootstrap nodes
+                # Bootstrap nodes have nodeid -1, other nodes have positive
+                # node ids
+                next if broker_statistics['nodeid'] == -1
+
+                public_send(
+                  metric.type,
+                  metric.name,
+                  broker_statistics.dig(*metric.key_location),
+                  tags: default_tags + ["broker:#{broker_statistics['nodename']}"]
+                )
+              end
+            else
+              raise ArgumentError, metric.scope
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/waterdrop/instrumentation.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  # Namespace for all the things related with WaterDrop instrumentation process
+  module Instrumentation
+    class << self
+      # Builds a manager for statistics callbacks
+      # @return [WaterDrop::CallbacksManager]
+      def statistics_callbacks
+        @statistics_callbacks ||= CallbacksManager.new
+      end
+
+      # Builds a manager for error callbacks
+      # @return [WaterDrop::CallbacksManager]
+      def error_callbacks
+        @error_callbacks ||= CallbacksManager.new
+      end
+    end
+  end
+end

data/lib/waterdrop/patches/rdkafka/bindings.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Patches
+    module Rdkafka
+      # Extends `Rdkafka::Bindings` with some extra methods and updates callbacks that we intend
+      # to work with in a bit different way than rdkafka itself
+      module Bindings
+        class << self
+          # Add extra methods that we need
+          # @param mod [::Rdkafka::Bindings] rdkafka bindings module
+          def included(mod)
+            mod.attach_function :rd_kafka_name, [:pointer], :string
+
+            # Default rdkafka setup for errors doest not propagate client details, thus it always
+            # publishes all the stuff for all rdkafka instances. We change that by providing
+            # function that fetches the instance name, allowing us to have better notifications
+            mod.send(:remove_const, :ErrorCallback)
+            mod.const_set(:ErrorCallback, build_error_callback)
+          end
+
+          # @return [FFI::Function] overwritten callback function
+          def build_error_callback
+            FFI::Function.new(
+              :void, %i[pointer int string pointer]
+            ) do |client_prr, err_code, reason, _opaque|
+              return nil unless ::Rdkafka::Config.error_callback
+
+              name = ::Rdkafka::Bindings.rd_kafka_name(client_prr)
+
+              error = ::Rdkafka::RdkafkaError.new(err_code, broker_message: reason)
+
+              ::Rdkafka::Config.error_callback.call(name, error)
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
+::Rdkafka::Bindings.include(::WaterDrop::Patches::Rdkafka::Bindings)