waterdrop 2.0.3 → 2.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e04020bed5a4adb3cb5ed3486367c9bae0b66998b0c25912a699b64e5c348917
-  data.tar.gz: c2ce9c2f9a3169201a66f948b307ab5d4b1c0d5caa130b1dca1e7c2f90db666e
+  metadata.gz: 261d7b8295e2e1b3a6b9658d4409dc58ff8c336e8e5d6064e80d92c8e3215bbe
+  data.tar.gz: f67bad60fe2cd37ac1658533ceec0daa1ec9ab527bb022da7f1cc40ccde89088
 SHA512:
-  metadata.gz: a40f4811e0ae0356980dfec3a0c0847409186b2afcb7d148e3afadb2c879c6ead03fcfe365e2a3cd15ed45ea1bf9afef7bd89567918dfde3d13741044f5ad36e
-  data.tar.gz: ec09878e1db68243141b4d2a751b7841627f38866199680ed8e08910257e28862c620864d5bd27c880e3176c960960040ef37c447841c00718fd6419f56770f1
+  metadata.gz: e0529057c22d4b76c61f788f038e8fb024785aa3a2c89c266cfe2b50d1f2b44e8a96e16ca626f62e721d4ae48fe4afac7113e32e5cd5ccbc5e7adf36aa0558af
+  data.tar.gz: 31241cad722c315aefab9d4ad2613908f053bf3c905779e8fe5c6f1975fbc866b34526442386c1faec2e2386d5f5332c4b7e80a326aa8aca9b8ab1b94e51d572

data/.github/workflows/ci.yml

@@ -17,7 +17,7 @@ jobs:
           - '3.0'
           - '2.7'
           - '2.6'
-          - 'jruby-head'
+          - 'jruby-9.3.1.0'
         include:
           - ruby: '3.0'
             coverage: 'true'

data/CHANGELOG.md

@@ -1,5 +1,39 @@
 # WaterDrop changelog
 
+## 2.0.7 (2021-12-03)
+- Source code metadata url added to the gemspec
+- Replace `:producer` with `:producer_id` in events and update `StdoutListener` accordingly. This change aligns all the events in terms of not publishing the whole producer object in the events.
+- Add `error.emitted` into the `StdoutListener`.
+- Enable `StdoutLogger` in specs for additional integration coverage.
+
+## 2.0.6 (2021-12-01)
+- #218 - Fixes a case, where dispatch of callbacks the same moment a new producer was created could cause a concurrency issue in the manager.
+- Fix some unstable specs.
+
+## 2.0.5 (2021-11-28)
+
+### Bug fixes
+
+- Fixes an issue where multiple producers would emit stats of other producers causing the same stats to be published several times (as many times as a number of producers). This could cause invalid reporting for multi-kafka setups.
+- Fixes a bug where emitted statistics would contain their first value as the first delta value for first stats emitted.
+- Fixes a bug where decorated statistics would include a delta for a root field with non-numeric values.
+
+### Changes and features
+- Introduces support for error callbacks instrumentation notifications with `error.emitted` monitor emitted key for tracking background errors that would occur on the producer (disconnects, etc).
+- Removes the `:producer` key from `statistics.emitted` and replaces it with `:producer_id` not to inject whole producer into the payload
+- Removes the `:producer` key from `message.acknowledged` and replaces it with `:producer_id` not to inject whole producer into the payload
+- Cleanup and refactor of callbacks support to simplify the API and make it work with Rdkafka way of things.
+- Introduces a callbacks manager concept that will also be within in Karafka `2.0` for both statistics and errors tracking per client.
+- Sets default Kafka `client.id` to `waterdrop` when not set.
+- Updates specs to always emit statistics for better test coverage.
+- Adds statistics and errors integration specs running against Kafka.
+- Replaces direct `RSpec.describe` reference with auto-discovery
+- Patches `rdkafka` to provide functionalities that are needed for granular callback support.
+
+## 2.0.4 (2021-09-19)
+- Update `dry-*` to the recent versions and update settings syntax to match it
+- Update Zeitwerk requirement
+
 ## 2.0.3 (2021-09-05)
 - Remove rdkafka patch in favour of spec topic pre-creation
 - Do not close client that was never used upon closing producer

data/Gemfile.lock

@@ -1,13 +1,13 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.0.3)
+    waterdrop (2.0.7)
       concurrent-ruby (>= 1.1)
-      dry-configurable (~> 0.8)
-      dry-monitor (~> 0.3)
-      dry-validation (~> 1.3)
-      rdkafka (>= 0.6.0)
-      zeitwerk (~> 2.1)
+      dry-configurable (~> 0.13)
+      dry-monitor (~> 0.5)
+      dry-validation (~> 1.7)
+      rdkafka (>= 0.10)
+      zeitwerk (~> 2.3)
 
 GEM
   remote: https://rubygems.org/
@@ -22,15 +22,14 @@ GEM
     concurrent-ruby (1.1.9)
     diff-lcs (1.4.4)
     docile (1.4.0)
-    dry-configurable (0.12.1)
+    dry-configurable (0.13.0)
       concurrent-ruby (~> 1.0)
-      dry-core (~> 0.5, >= 0.5.0)
-    dry-container (0.8.0)
+      dry-core (~> 0.6)
+    dry-container (0.9.0)
       concurrent-ruby (~> 1.0)
-      dry-configurable (~> 0.1, >= 0.1.3)
+      dry-configurable (~> 0.13, >= 0.13.0)
     dry-core (0.7.1)
       concurrent-ruby (~> 1.0)
-    dry-equalizer (0.3.0)
     dry-events (0.3.0)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.5, >= 0.5)
@@ -39,13 +38,13 @@ GEM
     dry-logic (1.2.0)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.5, >= 0.5)
-    dry-monitor (0.4.0)
-      dry-configurable (~> 0.5)
+    dry-monitor (0.5.0)
+      dry-configurable (~> 0.13, >= 0.13.0)
       dry-core (~> 0.5, >= 0.5)
       dry-events (~> 0.2)
-    dry-schema (1.7.1)
+    dry-schema (1.8.0)
       concurrent-ruby (~> 1.0)
-      dry-configurable (~> 0.8, >= 0.8.3)
+      dry-configurable (~> 0.13, >= 0.13.0)
       dry-core (~> 0.5, >= 0.5)
       dry-initializer (~> 3.0)
       dry-logic (~> 1.0)
@@ -56,25 +55,24 @@ GEM
       dry-core (~> 0.5, >= 0.5)
       dry-inflector (~> 0.1, >= 0.1.2)
       dry-logic (~> 1.0, >= 1.0.2)
-    dry-validation (1.6.0)
+    dry-validation (1.7.0)
       concurrent-ruby (~> 1.0)
       dry-container (~> 0.7, >= 0.7.1)
-      dry-core (~> 0.4)
-      dry-equalizer (~> 0.2)
+      dry-core (~> 0.5, >= 0.5)
       dry-initializer (~> 3.0)
-      dry-schema (~> 1.5, >= 1.5.2)
+      dry-schema (~> 1.8, >= 1.8.0)
     factory_bot (6.2.0)
       activesupport (>= 5.0.0)
-    ffi (1.15.3)
-    i18n (1.8.10)
+    ffi (1.15.4)
+    i18n (1.8.11)
       concurrent-ruby (~> 1.0)
-    mini_portile2 (2.7.0)
+    mini_portile2 (2.7.1)
     minitest (5.14.4)
     rake (13.0.6)
-    rdkafka (0.9.0)
-      ffi (~> 1.9)
-      mini_portile2 (~> 2.1)
-      rake (>= 12.3)
+    rdkafka (0.11.0)
+      ffi (~> 1.15)
+      mini_portile2 (~> 2.7)
+      rake (> 12)
     rspec (3.10.0)
       rspec-core (~> 3.10.0)
       rspec-expectations (~> 3.10.0)
@@ -87,7 +85,7 @@ GEM
     rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
-    rspec-support (3.10.2)
+    rspec-support (3.10.3)
     simplecov (0.21.2)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
@@ -96,7 +94,7 @@ GEM
     simplecov_json_formatter (0.1.3)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    zeitwerk (2.4.2)
+    zeitwerk (2.5.1)
 
 PLATFORMS
   x86_64-darwin
@@ -111,4 +109,4 @@ DEPENDENCIES
   waterdrop!
 
 BUNDLED WITH
-   2.2.26
+   2.2.31

data/README.md

@@ -8,7 +8,7 @@ Please refer to [this](https://github.com/karafka/waterdrop/tree/1.4) branch and
 
 [![Build Status](https://github.com/karafka/waterdrop/workflows/ci/badge.svg)](https://github.com/karafka/waterdrop/actions?query=workflow%3Aci)
 [![Gem Version](https://badge.fury.io/rb/waterdrop.svg)](http://badge.fury.io/rb/waterdrop)
-[![Join the chat at https://gitter.im/karafka/karafka](https://badges.gitter.im/karafka/karafka.svg)](https://gitter.im/karafka/karafka)
+[![Join the chat at https://slack.karafka.io](https://raw.githubusercontent.com/karafka/misc/master/slack.svg)](https://slack.karafka.io)
 
 Gem used to send messages to Kafka in an easy way with an extra validation layer. It is a part of the [Karafka](https://github.com/karafka/karafka) ecosystem.
 
@@ -24,22 +24,20 @@ It:
 
 ## Table of contents
 
-- [WaterDrop](#waterdrop)
-  * [Table of contents](#table-of-contents)
-  * [Installation](#installation)
-  * [Setup](#setup)
-    + [WaterDrop configuration options](#waterdrop-configuration-options)
-    + [Kafka configuration options](#kafka-configuration-options)
-  * [Usage](#usage)
-    + [Basic usage](#basic-usage)
-    + [Buffering](#buffering)
-      - [Using WaterDrop to buffer messages based on the application logic](#using-waterdrop-to-buffer-messages-based-on-the-application-logic)
-      - [Using WaterDrop with rdkafka buffers to achieve periodic auto-flushing](#using-waterdrop-with-rdkafka-buffers-to-achieve-periodic-auto-flushing)
-  * [Instrumentation](#instrumentation)
-    + [Usage statistics](#usage-statistics)
-    + [Forking and potential memory problems](#forking-and-potential-memory-problems)
-  * [References](#references)
-  * [Note on contributions](#note-on-contributions)
+- [Installation](#installation)
+- [Setup](#setup)
+  * [WaterDrop configuration options](#waterdrop-configuration-options)
+  * [Kafka configuration options](#kafka-configuration-options)
+- [Usage](#usage)
+  * [Basic usage](#basic-usage)
+  * [Buffering](#buffering)
+      + [Using WaterDrop to buffer messages based on the application logic](#using-waterdrop-to-buffer-messages-based-on-the-application-logic)
+      + [Using WaterDrop with rdkafka buffers to achieve periodic auto-flushing](#using-waterdrop-with-rdkafka-buffers-to-achieve-periodic-auto-flushing)
+- [Instrumentation](#instrumentation)
+  * [Usage statistics](#usage-statistics)
+  * [Error notifications](#error-notifications)
+  * [Forking and potential memory problems](#forking-and-potential-memory-problems)
+- [Note on contributions](#note-on-contributions)
 
 ## Installation
 
@@ -290,25 +288,46 @@ producer.close
 
 Note: The metrics returned may not be completely consistent between brokers, toppars and totals, due to the internal asynchronous nature of librdkafka. E.g., the top level tx total may be less than the sum of the broker tx values which it represents.
 
+### Error notifications
+
+Aside from errors related to publishing messages like `buffer.flushed_async.error`, WaterDrop allows you to listen to errors that occur in its internal background threads. Things like reconnecting to Kafka upon network errors and others unrelated to publishing messages are all available under `error.emitted` notification key. You can subscribe to this event to ensure your setup is healthy and without any problems that would otherwise go unnoticed as long as messages are delivered.
+
+```ruby
+producer = WaterDrop::Producer.new do |config|
+  # Note invalid connection port...
+  config.kafka = { 'bootstrap.servers': 'localhost:9090' }
+end
+
+producer.monitor.subscribe('error.emitted') do |event|
+  error = event[:error]
+
+  p "Internal error occurred: #{error}"
+end
+
+# Run this code without Kafka cluster
+loop do
+  producer.produce_async(topic: 'events', payload: 'data')
+
+  sleep(1)
+end
+
+# After you stop your Kafka cluster, you will see a lot of those:
+#
+# Internal error occurred: Local: Broker transport failure (transport)
+#
+# Internal error occurred: Local: Broker transport failure (transport)
+```
+
 ### Forking and potential memory problems
 
 If you work with forked processes, make sure you **don't** use the producer before the fork. You can easily configure the producer and then fork and use it.
 
 To tackle this [obstacle](https://github.com/appsignal/rdkafka-ruby/issues/15) related to rdkafka, WaterDrop adds finalizer to each of the producers to close the rdkafka client before the Ruby process is shutdown. Due to the [nature of the finalizers](https://www.mikeperham.com/2010/02/24/the-trouble-with-ruby-finalizers/), this implementation prevents producers from being GCed (except upon VM shutdown) and can cause memory leaks if you don't use persistent/long-lived producers in a long-running process or if you don't use the `#close` method of a producer when it is no longer needed. Creating a producer instance for each message is anyhow a rather bad idea, so we recommend not to.
 
-## References
-
-* [WaterDrop code documentation](https://www.rubydoc.info/github/karafka/waterdrop)
-* [Karafka framework](https://github.com/karafka/karafka)
-* [WaterDrop Actions CI](https://github.com/karafka/waterdrop/actions?query=workflow%3Ac)
-* [WaterDrop Coditsu](https://app.coditsu.io/karafka/repositories/waterdrop)
-
 ## Note on contributions
 
-First, thank you for considering contributing to WaterDrop! It's people like you that make the open source community such a great community!
-
-Each pull request must pass all the RSpec specs and meet our quality requirements.
+First, thank you for considering contributing to the Karafka ecosystem! It's people like you that make the open source community such a great community!
 
-To check if everything is as it should be, we use [Coditsu](https://coditsu.io) that combines multiple linters and code analyzers for both code and documentation. Once you're done with your changes, submit a pull request.
+Each pull request must pass all the RSpec specs, integration tests and meet our quality requirements.
 
-Coditsu will automatically check your work against our quality standards. You can find your commit check results on the [builds page](https://app.coditsu.io/karafka/repositories/waterdrop/builds/commit_builds) of WaterDrop repository.
+Fork it, update and wait for the Github Actions results.

data/lib/water_drop/config.rb

@@ -7,33 +7,53 @@ module WaterDrop
   class Config
     include Dry::Configurable
 
+    # Defaults for kafka settings, that will be overwritten only if not present already
+    KAFKA_DEFAULTS = {
+      'client.id' => 'waterdrop'
+    }.freeze
+
+    private_constant :KAFKA_DEFAULTS
+
     # WaterDrop options
     #
     # option [String] id of the producer. This can be helpful when building producer specific
-    #   instrumentation or loggers. It is not the kafka producer id
-    setting(:id, false) { |id| id || SecureRandom.uuid }
+    #   instrumentation or loggers. It is not the kafka client id. It is an id that should be
+    #   unique for each of the producers
+    setting(
+      :id,
+      default: false,
+      constructor: ->(id) { id || SecureRandom.uuid }
+    )
     # option [Instance] logger that we want to use
     # @note Due to how rdkafka works, this setting is global for all the producers
-    setting(:logger, false) { |logger| logger || Logger.new($stdout, level: Logger::WARN) }
+    setting(
+      :logger,
+      default: false,
+      constructor: ->(logger) { logger || Logger.new($stdout, level: Logger::WARN) }
+    )
     # option [Instance] monitor that we want to use. See instrumentation part of the README for
     #   more details
-    setting(:monitor, false) { |monitor| monitor || WaterDrop::Instrumentation::Monitor.new }
+    setting(
+      :monitor,
+      default: false,
+      constructor: ->(monitor) { monitor || WaterDrop::Instrumentation::Monitor.new }
+    )
     # option [Integer] max payload size allowed for delivery to Kafka
-    setting :max_payload_size, 1_000_012
+    setting :max_payload_size, default: 1_000_012
     # option [Integer] Wait that long for the delivery report or raise an error if this takes
     #   longer than the timeout.
-    setting :max_wait_timeout, 5
+    setting :max_wait_timeout, default: 5
     # option [Numeric] how long should we wait between re-checks on the availability of the
     #   delivery report. In a really robust systems, this describes the min-delivery time
     #   for a single sync message when produced in isolation
-    setting :wait_timeout, 0.005 # 5 milliseconds
+    setting :wait_timeout, default: 0.005 # 5 milliseconds
     # option [Boolean] should we send messages. Setting this to false can be really useful when
     #   testing and or developing because when set to false, won't actually ping Kafka but will
     #   run all the validations, etc
-    setting :deliver, true
+    setting :deliver, default: true
     # rdkafka options
     # @see https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
-    setting :kafka, {}
+    setting :kafka, default: {}
 
     # Configuration method
     # @yield Runs a block of code providing a config singleton instance to it
@@ -41,12 +61,28 @@ module WaterDrop
     def setup
       configure do |config|
         yield(config)
+
+        merge_kafka_defaults!(config)
         validate!(config.to_h)
+
+        ::Rdkafka::Config.logger = config.logger
       end
     end
 
     private
 
+    # Propagates the kafka setting defaults unless they are already present
+    # This makes it easier to set some values that users usually don't change but still allows them
+    # to overwrite the whole hash if they want to
+    # @param config [Dry::Configurable::Config] dry config of this producer
+    def merge_kafka_defaults!(config)
+      KAFKA_DEFAULTS.each do |key, value|
+        next if config.kafka.key?(key)
+
+        config.kafka[key] = value
+      end
+    end
+
     # Validates the configuration and if anything is wrong, will raise an exception
     # @param config_hash [Hash] config hash with setup details
     # @raise [WaterDrop::Errors::ConfigurationInvalidError] raised when something is wrong with

data/lib/water_drop/instrumentation/callbacks/delivery.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Instrumentation
+    module Callbacks
+      # Creates a callable that we want to run upon each message delivery or failure
+      #
+      # @note We don't have to provide client_name here as this callback is per client instance
+      class Delivery
+        # @param producer_id [String] id of the current producer
+        # @param monitor [WaterDrop::Instrumentation::Monitor] monitor we are using
+        def initialize(producer_id, monitor)
+          @producer_id = producer_id
+          @monitor = monitor
+        end
+
+        # Emits delivery details to the monitor
+        # @param delivery_report [Rdkafka::Producer::DeliveryReport] delivery report
+        def call(delivery_report)
+          @monitor.instrument(
+            'message.acknowledged',
+            producer_id: @producer_id,
+            offset: delivery_report.offset,
+            partition: delivery_report.partition
+          )
+        end
+      end
+    end
+  end
+end

data/lib/water_drop/instrumentation/callbacks/error.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Instrumentation
+    module Callbacks
+      # Callback that kicks in when error occurs and is published in a background thread
+      class Error
+        # @param producer_id [String] id of the current producer
+        # @param client_name [String] rdkafka client name
+        # @param monitor [WaterDrop::Instrumentation::Monitor] monitor we are using
+        def initialize(producer_id, client_name, monitor)
+          @producer_id = producer_id
+          @client_name = client_name
+          @monitor = monitor
+        end
+
+        # Runs the instrumentation monitor with error
+        # @param client_name [String] rdkafka client name
+        # @param error [Rdkafka::Error] error that occurred
+        # @note It will only instrument on errors of the client of our producer
+        def call(client_name, error)
+          # Emit only errors related to our client
+          # Same as with statistics (mor explanation there)
+          return unless @client_name == client_name
+
+          @monitor.instrument(
+            'error.emitted',
+            producer_id: @producer_id,
+            error: error
+          )
+        end
+      end
+    end
+  end
+end

data/lib/water_drop/instrumentation/callbacks/statistics.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Instrumentation
+    # Namespace for handlers of callbacks emitted by the kafka client lib
+    module Callbacks
+      # Statistics callback handler
+      # @note We decorate the statistics with our own decorator because some of the metrics from
+      #   rdkafka are absolute. For example number of sent messages increases not in reference to
+      #   previous statistics emit but from the beginning of the process. We decorate it with diff
+      #   of all the numeric values against the data from the previous callback emit
+      class Statistics
+        # @param producer_id [String] id of the current producer
+        # @param client_name [String] rdkafka client name
+        # @param monitor [WaterDrop::Instrumentation::Monitor] monitor we are using
+        def initialize(producer_id, client_name, monitor)
+          @producer_id = producer_id
+          @client_name = client_name
+          @monitor = monitor
+          @statistics_decorator = StatisticsDecorator.new
+        end
+
+        # Emits decorated statistics to the monitor
+        # @param statistics [Hash] rdkafka statistics
+        def call(statistics)
+          # Emit only statistics related to our client
+          # rdkafka does not have per-instance statistics hook, thus we need to make sure that we
+          # emit only stats that are related to current producer. Otherwise we would emit all of
+          # all the time.
+          return unless @client_name == statistics['name']
+
+          @monitor.instrument(
+            'statistics.emitted',
+            producer_id: @producer_id,
+            statistics: @statistics_decorator.call(statistics)
+          )
+        end
+      end
+    end
+  end
+end

data/lib/water_drop/instrumentation/callbacks/statistics_decorator.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Instrumentation
+    module Callbacks
+      # Many of the librdkafka statistics are absolute values instead of a gauge.
+      # This means, that for example number of messages sent is an absolute growing value
+      # instead of being a value of messages sent from the last statistics report.
+      # This decorator calculates the diff against previously emited stats, so we get also
+      # the diff together with the original values
+      class StatisticsDecorator
+        def initialize
+          @previous = {}.freeze
+        end
+
+        # @param emited_stats [Hash] original emited statistics
+        # @return [Hash] emited statistics extended with the diff data
+        # @note We modify the emited statistics, instead of creating new. Since we don't expose
+        #   any API to get raw data, users can just assume that the result of this decoration is
+        #   the proper raw stats that they can use
+        def call(emited_stats)
+          diff(
+            @previous,
+            emited_stats
+          )
+
+          @previous = emited_stats
+
+          emited_stats.freeze
+        end
+
+        private
+
+        # Calculates the diff of the provided values and modifies in place the emited statistics
+        #
+        # @param previous [Object] previous value from the given scope in which
+        #   we are
+        # @param current [Object] current scope from emitted statistics
+        # @return [Object] the diff if the values were numerics or the current scope
+        def diff(previous, current)
+          if current.is_a?(Hash)
+            # @note We cannot use #each_key as we modify the content of the current scope
+            #   in place (in case it's a hash)
+            current.keys.each do |key|
+              append(
+                current,
+                key,
+                diff((previous || {})[key], (current || {})[key])
+              )
+            end
+          end
+
+          # Diff can be computed only for numerics
+          return current unless current.is_a?(Numeric)
+          # If there was no previous value, delta is always zero
+          return 0 unless previous
+          # Should never happen but just in case, a type changed in between stats
+          return current unless previous.is_a?(Numeric)
+
+          current - previous
+        end
+
+        # Appends the result of the diff to a given key as long as the result is numeric
+        #
+        # @param current [Hash] current scope
+        # @param key [Symbol] key based on which we were diffing
+        # @param result [Object] diff result
+        def append(current, key, result)
+          return unless result.is_a?(Numeric)
+          return if current.frozen?
+
+          current["#{key}_d"] = result
+        end
+      end
+    end
+  end
+end

data/lib/water_drop/instrumentation/callbacks_manager.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Instrumentation
+    # This manager allows us to register multiple callbacks into a hook that is suppose to support
+    # a single callback
+    class CallbacksManager
+      # @return [::WaterDrop::Instrumentation::CallbacksManager]
+      def initialize
+        @callbacks = Concurrent::Hash.new
+      end
+
+      # Invokes all the callbacks registered one after another
+      #
+      # @param args [Object] any args that should go to the callbacks
+      # @note We do not use `#each_value` here on purpose. With it being used, we cannot dispatch
+      #   callbacks and add new at the same time. Since we don't know when and in what thread
+      #   things are going to be added to the manager, we need to extract values into an array and
+      #   run it. That way we can add new things the same time.
+      def call(*args)
+        @callbacks.values.each { |callback| callback.call(*args) }
+      end
+
+      # Adds a callback to the manager
+      #
+      # @param id [String] id of the callback (used when deleting it)
+      # @param callable [#call] object that responds to a `#call` method
+      def add(id, callable)
+        @callbacks[id] = callable
+      end
+
+      # Removes the callback from the manager
+      # @param id [String] id of the callback we want to remove
+      def delete(id)
+        @callbacks.delete(id)
+      end
+    end
+  end
+end

data/lib/water_drop/instrumentation/monitor.rb

@@ -13,18 +13,24 @@ module WaterDrop
       # @note The non-error once support timestamp benchmarking
       EVENTS = %w[
         producer.closed
+
         message.produced_async
         message.produced_sync
+        message.acknowledged
+        message.buffered
+
         messages.produced_async
         messages.produced_sync
-        message.buffered
         messages.buffered
-        message.acknowledged
+
         buffer.flushed_async
         buffer.flushed_async.error
         buffer.flushed_sync
         buffer.flushed_sync.error
+
         statistics.emitted
+
+        error.emitted
       ].freeze
 
       private_constant :EVENTS

data/lib/water_drop/instrumentation/stdout_listener.rb

@@ -51,7 +51,7 @@ module WaterDrop
         message = event[:message]
 
         info(event, "Buffering of a message to '#{message[:topic]}' topic")
-        debug(event, [message, event[:producer].messages.size])
+        debug(event, [message])
       end
 
       # @param event [Dry::Events::Event] event that happened with the details
@@ -59,7 +59,7 @@ module WaterDrop
         messages = event[:messages]
 
         info(event, "Buffering of #{messages.size} messages")
-        debug(event, [messages, event[:producer].messages.size])
+        debug(event, [messages, messages.size])
       end
 
       # @param event [Dry::Events::Event] event that happened with the details
@@ -99,7 +99,15 @@ module WaterDrop
       # @param event [Dry::Events::Event] event that happened with the details
       def on_producer_closed(event)
         info event, 'Closing producer'
-        debug event, event[:producer].messages.size
+        debug event, ''
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the error details
+      def on_error_emitted(event)
+        error = event[:error]
+
+        error(event, "Background thread error emitted: #{error}")
+        debug(event, '')
       end
 
       private
@@ -107,19 +115,19 @@ module WaterDrop
       # @param event [Dry::Events::Event] event that happened with the details
       # @param log_message [String] message we want to publish
       def debug(event, log_message)
-        @logger.debug("[#{event[:producer].id}] #{log_message}")
+        @logger.debug("[#{event[:producer_id]}] #{log_message}")
       end
 
       # @param event [Dry::Events::Event] event that happened with the details
       # @param log_message [String] message we want to publish
       def info(event, log_message)
-        @logger.info("[#{event[:producer].id}] #{log_message} took #{event[:time]} ms")
+        @logger.info("[#{event[:producer_id]}] #{log_message} took #{event[:time]} ms")
       end
 
       # @param event [Dry::Events::Event] event that happened with the details
       # @param log_message [String] message we want to publish
       def error(event, log_message)
-        @logger.error("[#{event[:producer].id}] #{log_message}")
+        @logger.error("[#{event[:producer_id]}] #{log_message}")
       end
     end
   end

data/lib/water_drop/instrumentation.rb

@@ -3,5 +3,18 @@
 module WaterDrop
   # Namespace for all the things related with WaterDrop instrumentation process
   module Instrumentation
+    class << self
+      # Builds a manager for statistics callbacks
+      # @return [WaterDrop::CallbacksManager]
+      def statistics_callbacks
+        @statistics_callbacks ||= CallbacksManager.new
+      end
+
+      # Builds a manager for error callbacks
+      # @return [WaterDrop::CallbacksManager]
+      def error_callbacks
+        @error_callbacks ||= CallbacksManager.new
+      end
+    end
   end
 end

data/lib/water_drop/patches/rdkafka/bindings.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Patches
+    module Rdkafka
+      # Extends `Rdkafka::Bindings` with some extra methods and updates callbacks that we intend
+      # to work with in a bit different way than rdkafka itself
+      module Bindings
+        class << self
+          # Add extra methods that we need
+          # @param mod [::Rdkafka::Bindings] rdkafka bindings module
+          def included(mod)
+            mod.attach_function :rd_kafka_name, [:pointer], :string
+
+            # Default rdkafka setup for errors doest not propagate client details, thus it always
+            # publishes all the stuff for all rdkafka instances. We change that by providing
+            # function that fetches the instance name, allowing us to have better notifications
+            mod.send(:remove_const, :ErrorCallback)
+            mod.const_set(:ErrorCallback, build_error_callback)
+          end
+
+          # @return [FFI::Function] overwritten callback function
+          def build_error_callback
+            FFI::Function.new(
+              :void, %i[pointer int string pointer]
+            ) do |client_prr, err_code, reason, _opaque|
+              return nil unless ::Rdkafka::Config.error_callback
+
+              name = ::Rdkafka::Bindings.rd_kafka_name(client_prr)
+
+              error = ::Rdkafka::RdkafkaError.new(err_code, broker_message: reason)
+
+              ::Rdkafka::Config.error_callback.call(name, error)
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
+::Rdkafka::Bindings.include(::WaterDrop::Patches::Rdkafka::Bindings)

data/lib/water_drop/patches/rdkafka/producer.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  # Patches to external components
+  module Patches
+    # Rdkafka related patches
+    module Rdkafka
+      # Rdkafka::Producer patches
+      module Producer
+        # Adds a method that allows us to get the native kafka producer name
+        # @return [String] producer instance name
+        def name
+          ::Rdkafka::Bindings.rd_kafka_name(@native_kafka)
+        end
+      end
+    end
+  end
+end
+
+::Rdkafka::Producer.include ::WaterDrop::Patches::Rdkafka::Producer

data/lib/water_drop/producer/async.rb

@@ -19,7 +19,7 @@ module WaterDrop
 
         @monitor.instrument(
           'message.produced_async',
-          producer: self,
+          producer_id: id,
           message: message
         ) { client.produce(**message) }
       end
@@ -40,7 +40,7 @@ module WaterDrop
 
         @monitor.instrument(
           'messages.produced_async',
-          producer: self,
+          producer_id: id,
           messages: messages
         ) do
           messages.map { |message| client.produce(**message) }

data/lib/water_drop/producer/buffer.rb

@@ -23,7 +23,7 @@ module WaterDrop
 
         @monitor.instrument(
           'message.buffered',
-          producer: self,
+          producer_id: id,
           message: message
         ) { @messages << message }
       end
@@ -40,7 +40,7 @@ module WaterDrop
 
         @monitor.instrument(
           'messages.buffered',
-          producer: self,
+          producer_id: id,
           messages: messages
         ) do
           messages.each { |message| @messages << message }
@@ -56,7 +56,7 @@ module WaterDrop
 
         @monitor.instrument(
           'buffer.flushed_async',
-          producer: self,
+          producer_id: id,
           messages: @messages
         ) { flush(false) }
       end
@@ -69,7 +69,7 @@ module WaterDrop
 
         @monitor.instrument(
           'buffer.flushed_sync',
-          producer: self,
+          producer_id: id,
           messages: @messages
         ) { flush(true) }
       end
@@ -104,7 +104,7 @@ module WaterDrop
         end
       rescue *RESCUED_ERRORS => e
         key = sync ? 'buffer.flushed_sync.error' : 'buffer.flush_async.error'
-        @monitor.instrument(key, producer: self, error: e, dispatched: dispatched)
+        @monitor.instrument(key, producer_id: id, error: e, dispatched: dispatched)
 
         raise Errors::FlushFailureError.new(dispatched)
       end

data/lib/water_drop/producer/builder.rb

@@ -12,51 +12,16 @@ module WaterDrop
       def call(producer, config)
         return DummyClient.new unless config.deliver
 
-        Rdkafka::Config.logger = config.logger
-        Rdkafka::Config.statistics_callback = build_statistics_callback(producer, config.monitor)
-
         client = Rdkafka::Config.new(config.kafka.to_h).producer
-        client.delivery_callback = build_delivery_callback(producer, config.monitor)
-        client
-      end
 
-      private
+        # This callback is not global and is per client, thus we do not have to wrap it with a
+        # callbacks manager to make it work
+        client.delivery_callback = Instrumentation::Callbacks::Delivery.new(
+          producer.id,
+          config.monitor
+        )
 
-      # Creates a proc that we want to run upon each successful message delivery
-      #
-      # @param producer [Producer]
-      # @param monitor [Object] monitor we want to use
-      # @return [Proc] delivery callback
-      def build_delivery_callback(producer, monitor)
-        lambda do |delivery_report|
-          monitor.instrument(
-            'message.acknowledged',
-            producer: producer,
-            offset: delivery_report.offset,
-            partition: delivery_report.partition
-          )
-        end
-      end
-
-      # Creates a proc that we want to run upon each statistics callback execution
-      #
-      # @param producer [Producer]
-      # @param monitor [Object] monitor we want to use
-      # @return [Proc] statistics callback
-      # @note We decorate the statistics with our own decorator because some of the metrics from
-      #   rdkafka are absolute. For example number of sent messages increases not in reference to
-      #   previous statistics emit but from the beginning of the process. We decorate it with diff
-      #   of all the numeric values against the data from the previous callback emit
-      def build_statistics_callback(producer, monitor)
-        statistics_decorator = StatisticsDecorator.new
-
-        lambda do |statistics|
-          monitor.instrument(
-            'statistics.emitted',
-            producer: producer,
-            statistics: statistics_decorator.call(statistics)
-          )
-        end
+        client
       end
     end
   end

data/lib/water_drop/producer/sync.rb

@@ -21,7 +21,7 @@ module WaterDrop
 
         @monitor.instrument(
           'message.produced_sync',
-          producer: self,
+          producer_id: id,
           message: message
         ) do
           client
@@ -49,7 +49,7 @@ module WaterDrop
         ensure_active!
         messages.each { |message| validate_message!(message) }
 
-        @monitor.instrument('messages.produced_sync', producer: self, messages: messages) do
+        @monitor.instrument('messages.produced_sync', producer_id: id, messages: messages) do
           messages
             .map { |message| client.produce(**message) }
             .map! do |handler|

data/lib/water_drop/producer.rb

@@ -80,6 +80,19 @@ module WaterDrop
 
         @pid = Process.pid
         @client = Builder.new.call(self, @config)
+
+        # Register statistics runner for this particular type of callbacks
+        ::WaterDrop::Instrumentation.statistics_callbacks.add(
+          @id,
+          Instrumentation::Callbacks::Statistics.new(@id, @client.name, @config.monitor)
+        )
+
+        # Register error tracking callback
+        ::WaterDrop::Instrumentation.error_callbacks.add(
+          @id,
+          Instrumentation::Callbacks::Error.new(@id, @client.name, @config.monitor)
+        )
+
         @status.connected!
       end
 
@@ -93,7 +106,7 @@ module WaterDrop
 
         @monitor.instrument(
           'producer.closed',
-          producer: self
+          producer_id: id
         ) do
           @status.closing!
 
@@ -111,6 +124,10 @@ module WaterDrop
           # connection that anyhow would be immediately closed
           client.close if @client
 
+          # Remove callbacks runners that were registered
+          ::WaterDrop::Instrumentation.statistics_callbacks.delete(@id)
+          ::WaterDrop::Instrumentation.error_callbacks.delete(@id)
+
           @status.closed!
         end
       end

data/lib/water_drop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '2.0.3'
+  VERSION = '2.0.7'
 end

data/lib/water_drop.rb

@@ -28,3 +28,9 @@ Zeitwerk::Loader
   .tap { |loader| loader.ignore("#{__dir__}/waterdrop.rb") }
   .tap(&:setup)
   .tap(&:eager_load)
+
+# Rdkafka uses a single global callback for things. We bypass that by injecting a manager for
+# each callback type. Callback manager allows us to register more than one callback
+# @note Those managers are also used by Karafka for consumer related statistics
+Rdkafka::Config.statistics_callback = WaterDrop::Instrumentation.statistics_callbacks
+Rdkafka::Config.error_callback = WaterDrop::Instrumentation.error_callbacks

data/waterdrop.gemspec

@@ -11,17 +11,17 @@ Gem::Specification.new do |spec|
   spec.platform      = Gem::Platform::RUBY
   spec.authors       = ['Maciej Mensfeld']
   spec.email         = %w[maciej@mensfeld.pl]
-  spec.homepage      = 'https://github.com/karafka/waterdrop'
+  spec.homepage      = 'https://karafka.io'
   spec.summary       = 'Kafka messaging made easy!'
   spec.description   = spec.summary
   spec.license       = 'MIT'
 
   spec.add_dependency 'concurrent-ruby', '>= 1.1'
-  spec.add_dependency 'dry-configurable', '~> 0.8'
-  spec.add_dependency 'dry-monitor', '~> 0.3'
-  spec.add_dependency 'dry-validation', '~> 1.3'
-  spec.add_dependency 'rdkafka', '>= 0.6.0'
-  spec.add_dependency 'zeitwerk', '~> 2.1'
+  spec.add_dependency 'dry-configurable', '~> 0.13'
+  spec.add_dependency 'dry-monitor', '~> 0.5'
+  spec.add_dependency 'dry-validation', '~> 1.7'
+  spec.add_dependency 'rdkafka', '>= 0.10'
+  spec.add_dependency 'zeitwerk', '~> 2.3'
 
   spec.required_ruby_version = '>= 2.6.0'
 
@@ -33,4 +33,5 @@ Gem::Specification.new do |spec|
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = %w[lib]
+  spec.metadata      = { 'source_code_uri' => 'https://github.com/karafka/waterdrop' }
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 2.0.3
+  version: 2.0.7
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -34,7 +34,7 @@ cert_chain:
   R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
   pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
   -----END CERTIFICATE-----
-date: 2021-09-05 00:00:00.000000000 Z
+date: 2021-12-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -56,70 +56,70 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.13'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.13'
 - !ruby/object:Gem::Dependency
   name: dry-monitor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '0.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '0.5'
 - !ruby/object:Gem::Dependency
   name: dry-validation
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.7'
 - !ruby/object:Gem::Dependency
   name: rdkafka
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.6.0
+        version: '0.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.6.0
+        version: '0.10'
 - !ruby/object:Gem::Dependency
   name: zeitwerk
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '2.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '2.3'
 description: Kafka messaging made easy!
 email:
 - maciej@mensfeld.pl
@@ -129,7 +129,6 @@ extra_rdoc_files: []
 files:
 - ".coditsu/ci.yml"
 - ".diffend.yml"
-- ".github/FUNDING.yml"
 - ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rspec"
@@ -150,24 +149,31 @@ files:
 - lib/water_drop/contracts/message.rb
 - lib/water_drop/errors.rb
 - lib/water_drop/instrumentation.rb
+- lib/water_drop/instrumentation/callbacks/delivery.rb
+- lib/water_drop/instrumentation/callbacks/error.rb
+- lib/water_drop/instrumentation/callbacks/statistics.rb
+- lib/water_drop/instrumentation/callbacks/statistics_decorator.rb
+- lib/water_drop/instrumentation/callbacks_manager.rb
 - lib/water_drop/instrumentation/monitor.rb
 - lib/water_drop/instrumentation/stdout_listener.rb
+- lib/water_drop/patches/rdkafka/bindings.rb
+- lib/water_drop/patches/rdkafka/producer.rb
 - lib/water_drop/producer.rb
 - lib/water_drop/producer/async.rb
 - lib/water_drop/producer/buffer.rb
 - lib/water_drop/producer/builder.rb
 - lib/water_drop/producer/dummy_client.rb
-- lib/water_drop/producer/statistics_decorator.rb
 - lib/water_drop/producer/status.rb
 - lib/water_drop/producer/sync.rb
 - lib/water_drop/version.rb
 - lib/waterdrop.rb
 - log/.gitkeep
 - waterdrop.gemspec
-homepage: https://github.com/karafka/waterdrop
+homepage: https://karafka.io
 licenses:
 - MIT
-metadata: {}
+metadata:
+  source_code_uri: https://github.com/karafka/waterdrop
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -183,7 +189,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.26
+rubygems_version: 3.2.31
 signing_key: 
 specification_version: 4
 summary: Kafka messaging made easy!

data/.github/FUNDING.yml

@@ -1 +0,0 @@
-open_collective: karafka

data/lib/water_drop/producer/statistics_decorator.rb

@@ -1,71 +0,0 @@
-# frozen_string_literal: true
-
-module WaterDrop
-  class Producer
-    # Many of the librdkafka statistics are absolute values instead of a gauge.
-    # This means, that for example number of messages sent is an absolute growing value
-    # instead of being a value of messages sent from the last statistics report.
-    # This decorator calculates the diff against previously emited stats, so we get also
-    # the diff together with the original values
-    class StatisticsDecorator
-      def initialize
-        @previous = {}.freeze
-      end
-
-      # @param emited_stats [Hash] original emited statistics
-      # @return [Hash] emited statistics extended with the diff data
-      # @note We modify the emited statistics, instead of creating new. Since we don't expose
-      #   any API to get raw data, users can just assume that the result of this decoration is the
-      #   proper raw stats that they can use
-      def call(emited_stats)
-        diff(
-          @previous,
-          emited_stats
-        )
-
-        @previous = emited_stats
-
-        emited_stats.freeze
-      end
-
-      private
-
-      # Calculates the diff of the provided values and modifies in place the emited statistics
-      #
-      # @param previous [Object] previous value from the given scope in which
-      #   we are
-      # @param current [Object] current scope from emitted statistics
-      # @return [Object] the diff if the values were numerics or the current scope
-      def diff(previous, current)
-        if current.is_a?(Hash)
-          # @note We cannot use #each_key as we modify the content of the current scope
-          #   in place (in case it's a hash)
-          current.keys.each do |key|
-            append(
-              current,
-              key,
-              diff((previous || {})[key], (current || {})[key])
-            )
-          end
-        end
-
-        if current.is_a?(Numeric) && previous.is_a?(Numeric)
-          current - previous
-        else
-          current
-        end
-      end
-
-      # Appends the result of the diff to a given key as long as the result is numeric
-      #
-      # @param current [Hash] current scope
-      # @param key [Symbol] key based on which we were diffing
-      # @param result [Object] diff result
-      def append(current, key, result)
-        return unless result.is_a?(Numeric)
-
-        current["#{key}_d"] = result
-      end
-    end
-  end
-end