waterdrop 2.0.1 → 2.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 950a87d0cb91932f4fb6dc99dc45445948235db00d18ce03fca42f4d3262fed6
-  data.tar.gz: e90da26b2231c001af204d7f52de1078112b1617b38089a0e253ce8c90bb3baf
+  metadata.gz: 310a3d7e1a4d0e5825b3a01f59b29c22a9f180c639951763bdf936a23c1119fd
+  data.tar.gz: f6c0c498266ba067201e7983d5bdea7a0aee7810a403be1cd4f4b3d62ab60633
 SHA512:
-  metadata.gz: '0861a98c89e55cc8d0d00d15c9d251d09b4c83c27b87b0fd3b994be32eb285155fa786e8d79e61baf397e6f7eb995c05c0f6da4d4cfcf8658c56de0949d36c88'
-  data.tar.gz: 5527c9f42186a5764a2544252ffbf7d752842ac1c389a24d1e83a2c98744f90a3564776ff72d4ef324fa70d7c8ed6e07f05c0f1bb4e09daa8ad85859322983e8
+  metadata.gz: 4e486cfa6aa673e008eeaccb8cf920fbb30fce1d23277021d3c6a02e36ee14b8a280e9114b9be778bdb68ba4b07eb2d64371362c454c607edf3c4b57a26a0066
+  data.tar.gz: 50301b9c5a5e67434f46247b5d1a83e4af2577e0f3b8f251a2795bc48aaba8c59135025e606b8143ca57560a2eac6666c530bd5d1b6059ce2e61d008e1eb9385

data/.github/workflows/ci.yml

@@ -17,7 +17,7 @@ jobs:
           - '3.0'
           - '2.7'
           - '2.6'
-          - 'jruby-head'
+          - 'jruby-9.3.1.0'
         include:
           - ruby: '3.0'
             coverage: 'true'
@@ -29,6 +29,12 @@ jobs:
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{matrix.ruby}}
+      - name: Run Kafka with docker-compose
+        # We need to give Kafka enough time to start and create all the needed topics, etc
+        # If anyone has a better idea on how to do it smart and easily, please contact me
+        run: |
+          docker-compose up -d
+          sleep 5
       - name: Install latest bundler
         run: |
           gem install bundler --no-document
@@ -37,8 +43,6 @@ jobs:
         run: |
           bundle config set without development
           bundle install --jobs 4 --retry 3
-      - name: Run Kafka with docker-compose
-        run: docker-compose up -d
       - name: Run all tests
         env:
           GITHUB_COVERAGE: ${{matrix.coverage}}

data/.ruby-version

@@ -1 +1 @@
-3.0.1
+3.0.2

data/CHANGELOG.md

@@ -1,5 +1,38 @@
 # WaterDrop changelog
 
+## 2.0.5 (2021-11-28)
+
+### Bug fixes
+
+- Fixes an issue where multiple producers would emit stats of other producers causing the same stats to be published several times (as many times as a number of producers). This could cause invalid reporting for multi-kafka setups.
+- Fixes a bug where emitted statistics would contain their first value as the first delta value for first stats emitted.
+- Fixes a bug where decorated statistics would include a delta for a root field with non-numeric values.
+
+### Changes and features
+- Introduces support for error callbacks instrumentation notifications with `error.emitted` monitor emitted key for tracking background errors that would occur on the producer (disconnects, etc).
+- Removes the `:producer` key from `statistics.emitted` and replaces it with `:producer_id` not to inject whole producer into the payload
+- Removes the `:producer` key from `message.acknowledged` and replaces it with `:producer_id` not to inject whole producer into the payload
+- Cleanup and refactor of callbacks support to simplify the API and make it work with Rdkafka way of things.
+- Introduces a callbacks manager concept that will also be within in Karafka `2.0` for both statistics and errors tracking per client.
+- Sets default Kafka `client.id` to `waterdrop` when not set.
+- Updates specs to always emit statistics for better test coverage.
+- Adds statistics and errors integration specs running against Kafka.
+- Replaces direct `RSpec.describe` reference with auto-discovery
+- Patches `rdkafka` to provide functionalities that are needed for granular callback support.
+
+## 2.0.4 (2021-09-19)
+- Update `dry-*` to the recent versions and update settings syntax to match it
+- Update Zeitwerk requirement
+
+## 2.0.3 (2021-09-05)
+- Remove rdkafka patch in favour of spec topic pre-creation
+- Do not close client that was never used upon closing producer
+
+## 2.0.2 (2021-08-13)
+- Add support for `partition_key`
+- Switch license from `LGPL-3.0` to  `MIT`
+- Switch flushing on close to sync
+
 ## 2.0.1 (2021-06-05)
 - Remove Ruby 2.5 support and update minimum Ruby requirement to 2.6
 - Fix the `finalizer references object to be finalized` warning issued with 3.0

data/Gemfile.lock

@@ -1,18 +1,18 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.0.1)
+    waterdrop (2.0.5)
       concurrent-ruby (>= 1.1)
-      dry-configurable (~> 0.8)
-      dry-monitor (~> 0.3)
-      dry-validation (~> 1.3)
-      rdkafka (>= 0.6.0)
-      zeitwerk (~> 2.1)
+      dry-configurable (~> 0.13)
+      dry-monitor (~> 0.5)
+      dry-validation (~> 1.7)
+      rdkafka (>= 0.10)
+      zeitwerk (~> 2.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.1.3.2)
+    activesupport (6.1.4.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -22,30 +22,29 @@ GEM
     concurrent-ruby (1.1.9)
     diff-lcs (1.4.4)
     docile (1.4.0)
-    dry-configurable (0.12.1)
+    dry-configurable (0.13.0)
       concurrent-ruby (~> 1.0)
-      dry-core (~> 0.5, >= 0.5.0)
-    dry-container (0.7.2)
+      dry-core (~> 0.6)
+    dry-container (0.9.0)
       concurrent-ruby (~> 1.0)
-      dry-configurable (~> 0.1, >= 0.1.3)
-    dry-core (0.6.0)
+      dry-configurable (~> 0.13, >= 0.13.0)
+    dry-core (0.7.1)
       concurrent-ruby (~> 1.0)
-    dry-equalizer (0.3.0)
     dry-events (0.3.0)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.5, >= 0.5)
-    dry-inflector (0.2.0)
+    dry-inflector (0.2.1)
     dry-initializer (3.0.4)
     dry-logic (1.2.0)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.5, >= 0.5)
-    dry-monitor (0.4.0)
-      dry-configurable (~> 0.5)
+    dry-monitor (0.5.0)
+      dry-configurable (~> 0.13, >= 0.13.0)
       dry-core (~> 0.5, >= 0.5)
       dry-events (~> 0.2)
-    dry-schema (1.6.2)
+    dry-schema (1.8.0)
       concurrent-ruby (~> 1.0)
-      dry-configurable (~> 0.8, >= 0.8.3)
+      dry-configurable (~> 0.13, >= 0.13.0)
       dry-core (~> 0.5, >= 0.5)
       dry-initializer (~> 3.0)
       dry-logic (~> 1.0)
@@ -56,25 +55,24 @@ GEM
       dry-core (~> 0.5, >= 0.5)
       dry-inflector (~> 0.1, >= 0.1.2)
       dry-logic (~> 1.0, >= 1.0.2)
-    dry-validation (1.6.0)
+    dry-validation (1.7.0)
       concurrent-ruby (~> 1.0)
       dry-container (~> 0.7, >= 0.7.1)
-      dry-core (~> 0.4)
-      dry-equalizer (~> 0.2)
+      dry-core (~> 0.5, >= 0.5)
       dry-initializer (~> 3.0)
-      dry-schema (~> 1.5, >= 1.5.2)
+      dry-schema (~> 1.8, >= 1.8.0)
     factory_bot (6.2.0)
       activesupport (>= 5.0.0)
-    ffi (1.15.1)
-    i18n (1.8.10)
+    ffi (1.15.4)
+    i18n (1.8.11)
       concurrent-ruby (~> 1.0)
-    mini_portile2 (2.6.1)
+    mini_portile2 (2.7.1)
     minitest (5.14.4)
-    rake (13.0.3)
-    rdkafka (0.8.1)
-      ffi (~> 1.9)
-      mini_portile2 (~> 2.1)
-      rake (>= 12.3)
+    rake (13.0.6)
+    rdkafka (0.11.0)
+      ffi (~> 1.15)
+      mini_portile2 (~> 2.7)
+      rake (> 12)
     rspec (3.10.0)
       rspec-core (~> 3.10.0)
       rspec-expectations (~> 3.10.0)
@@ -87,7 +85,7 @@ GEM
     rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
-    rspec-support (3.10.2)
+    rspec-support (3.10.3)
     simplecov (0.21.2)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
@@ -96,11 +94,10 @@ GEM
     simplecov_json_formatter (0.1.3)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    zeitwerk (2.4.2)
+    zeitwerk (2.5.1)
 
 PLATFORMS
   x86_64-darwin
-  x86_64-darwin-19
   x86_64-linux
 
 DEPENDENCIES
@@ -112,4 +109,4 @@ DEPENDENCIES
   waterdrop!
 
 BUNDLED WITH
-   2.2.19
+   2.2.31

data/MIT-LICENSE

@@ -0,0 +1,18 @@
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -4,11 +4,11 @@
 
 WaterDrop `2.0` does **not** work with Karafka `1.*` and aims to either work as a standalone producer outside of Karafka `1.*` ecosystem or as a part of not yet released Karafka `2.0.*`.
 
-Please refer to [this](https://github.com/karafka/waterdrop/tree/1.4) branch and it's documentation for details about WaterDrop `1.*` usage.
+Please refer to [this](https://github.com/karafka/waterdrop/tree/1.4) branch and its documentation for details about WaterDrop `1.*` usage.
 
 [![Build Status](https://github.com/karafka/waterdrop/workflows/ci/badge.svg)](https://github.com/karafka/waterdrop/actions?query=workflow%3Aci)
 [![Gem Version](https://badge.fury.io/rb/waterdrop.svg)](http://badge.fury.io/rb/waterdrop)
-[![Join the chat at https://gitter.im/karafka/karafka](https://badges.gitter.im/karafka/karafka.svg)](https://gitter.im/karafka/karafka)
+[![Join the chat at https://slack.karafka.io](https://raw.githubusercontent.com/karafka/misc/master/slack.svg)](https://slack.karafka.io)
 
 Gem used to send messages to Kafka in an easy way with an extra validation layer. It is a part of the [Karafka](https://github.com/karafka/karafka) ecosystem.
 
@@ -20,7 +20,24 @@ It:
  - Supports buffering
  - Supports producing messages to multiple clusters
  - Supports multiple delivery policies
- - Works with Kafka 1.0+ and Ruby 2.5+
+ - Works with Kafka 1.0+ and Ruby 2.6+
+
+## Table of contents
+
+- [Installation](#installation)
+- [Setup](#setup)
+  * [WaterDrop configuration options](#waterdrop-configuration-options)
+  * [Kafka configuration options](#kafka-configuration-options)
+- [Usage](#usage)
+  * [Basic usage](#basic-usage)
+  * [Buffering](#buffering)
+      + [Using WaterDrop to buffer messages based on the application logic](#using-waterdrop-to-buffer-messages-based-on-the-application-logic)
+      + [Using WaterDrop with rdkafka buffers to achieve periodic auto-flushing](#using-waterdrop-with-rdkafka-buffers-to-achieve-periodic-auto-flushing)
+- [Instrumentation](#instrumentation)
+  * [Usage statistics](#usage-statistics)
+  * [Error notifications](#error-notifications)
+  * [Forking and potential memory problems](#forking-and-potential-memory-problems)
+- [Note on contributions](#note-on-contributions)
 
 ## Installation
 
@@ -44,8 +61,8 @@ bundle install
 
 WaterDrop is a complex tool, that contains multiple configuration options. To keep everything organized, all the configuration options were divided into two groups:
 
-- WaterDrop options - options directly related to Karafka framework and it's components
-- Kafka driver options - options related to `Kafka`
+- WaterDrop options - options directly related to WaterDrop and its components
+- Kafka driver options - options related to `rdkafka`
 
 To apply all those configuration options, you need to create a producer instance and use the ```#setup``` method:
 
@@ -131,20 +148,25 @@ Each message that you want to publish, will have its value checked.
 
 Here are all the things you can provide in the message hash:
 
-| Option      | Required | Value type    | Description                                           |
-|-------------|----------|---------------|-------------------------------------------------------|
-| `topic`     | true     | String        | The Kafka topic that should be written to             |
-| `payload`   | true     | String        | Data you want to send to Kafka                        |
-| `key`       | false    | String        | The key that should be set in the Kafka message       |
-| `partition` | false    | Integer       | A specific partition number that should be written to |
-| `timestamp` | false    | Time, Integer | The timestamp that should be set on the message       |
-| `headers`   | false    | Hash          | Headers for the message                               |
+| Option          | Required | Value type    | Description                                              |
+|-----------------|----------|---------------|----------------------------------------------------------|
+| `topic`         | true     | String        | The Kafka topic that should be written to                |
+| `payload`       | true     | String        | Data you want to send to Kafka                           |
+| `key`           | false    | String        | The key that should be set in the Kafka message          |
+| `partition`     | false    | Integer       | A specific partition number that should be written to    |
+| `partition_key` | false    | String        | Key to indicate the destination partition of the message |
+| `timestamp`     | false    | Time, Integer | The timestamp that should be set on the message          |
+| `headers`       | false    | Hash          | Headers for the message                                  |
 
 Keep in mind, that message you want to send should be either binary or stringified (to_s, to_json, etc).
 
 ### Buffering
 
-WaterDrop producers support buffering of messages, which means that you can easily implement periodic flushing for long running processes as well as buffer several messages to be flushed the same moment:
+WaterDrop producers support buffering messages in their internal buffers and on the `rdkafka` level via `queue.buffering.*` set of settings.
+
+This means that depending on your use case, you can achieve both granular buffering and flushing control when needed with context awareness and periodic and size-based flushing functionalities.
+
+#### Using WaterDrop to buffer messages based on the application logic
 
 ```ruby
 producer = WaterDrop::Producer.new
@@ -153,16 +175,41 @@ producer.setup do |config|
   config.kafka = { 'bootstrap.servers': 'localhost:9092' }
 end
 
-time = Time.now - 10
+# Simulating some events states of a transaction - notice, that the messages will be flushed to
+# kafka only upon arrival of the `finished` state.
+%w[
+  started
+  processed
+  finished
+].each do |state|
+  producer.buffer(topic: 'events', payload: state)
+
+  puts "The messages buffer size #{producer.messages.size}"
+  producer.flush_sync if state == 'finished'
+  puts "The messages buffer size #{producer.messages.size}"
+end
+
+producer.close
+```
+
+#### Using WaterDrop with rdkafka buffers to achieve periodic auto-flushing
+
+```ruby
+producer = WaterDrop::Producer.new
 
-while time < Time.now
-  time += 1
-  producer.buffer(topic: 'times', payload: Time.now.to_s)
+producer.setup do |config|
+  config.kafka = {
+    'bootstrap.servers': 'localhost:9092',
+    # Accumulate messages for at most 10 seconds
+    'queue.buffering.max.ms' => 10_000
+  }
 end
 
-puts "The messages buffer size #{producer.messages.size}"
-producer.flush_sync
-puts "The messages buffer size #{producer.message.size}"
+# WaterDrop will flush messages minimum once every 10 seconds
+30.times do |i|
+  producer.produce_async(topic: 'events', payload: i.to_s)
+  sleep(1)
+end
 
 producer.close
 ```
@@ -241,25 +288,46 @@ producer.close
 
 Note: The metrics returned may not be completely consistent between brokers, toppars and totals, due to the internal asynchronous nature of librdkafka. E.g., the top level tx total may be less than the sum of the broker tx values which it represents.
 
+### Error notifications
+
+Aside from errors related to publishing messages like `buffer.flushed_async.error`, WaterDrop allows you to listen to errors that occur in its internal background threads. Things like reconnecting to Kafka upon network errors and others unrelated to publishing messages are all available under `error.emitted` notification key. You can subscribe to this event to ensure your setup is healthy and without any problems that would otherwise go unnoticed as long as messages are delivered.
+
+```ruby
+producer = WaterDrop::Producer.new do |config|
+  # Note invalid connection port...
+  config.kafka = { 'bootstrap.servers': 'localhost:9090' }
+end
+
+producer.monitor.subscribe('error.emitted') do |event|
+  error = event[:error]
+
+  p "Internal error occurred: #{error}"
+end
+
+# Run this code without Kafka cluster
+loop do
+  producer.produce_async(topic: 'events', payload: 'data')
+
+  sleep(1)
+end
+
+# After you stop your Kafka cluster, you will see a lot of those:
+#
+# Internal error occurred: Local: Broker transport failure (transport)
+#
+# Internal error occurred: Local: Broker transport failure (transport)
+```
+
 ### Forking and potential memory problems
 
 If you work with forked processes, make sure you **don't** use the producer before the fork. You can easily configure the producer and then fork and use it.
 
 To tackle this [obstacle](https://github.com/appsignal/rdkafka-ruby/issues/15) related to rdkafka, WaterDrop adds finalizer to each of the producers to close the rdkafka client before the Ruby process is shutdown. Due to the [nature of the finalizers](https://www.mikeperham.com/2010/02/24/the-trouble-with-ruby-finalizers/), this implementation prevents producers from being GCed (except upon VM shutdown) and can cause memory leaks if you don't use persistent/long-lived producers in a long-running process or if you don't use the `#close` method of a producer when it is no longer needed. Creating a producer instance for each message is anyhow a rather bad idea, so we recommend not to.
 
-## References
-
-* [WaterDrop code documentation](https://www.rubydoc.info/github/karafka/waterdrop)
-* [Karafka framework](https://github.com/karafka/karafka)
-* [WaterDrop Actions CI](https://github.com/karafka/waterdrop/actions?query=workflow%3Ac)
-* [WaterDrop Coditsu](https://app.coditsu.io/karafka/repositories/waterdrop)
-
 ## Note on contributions
 
-First, thank you for considering contributing to WaterDrop! It's people like you that make the open source community such a great community!
-
-Each pull request must pass all the RSpec specs and meet our quality requirements.
+First, thank you for considering contributing to the Karafka ecosystem! It's people like you that make the open source community such a great community!
 
-To check if everything is as it should be, we use [Coditsu](https://coditsu.io) that combines multiple linters and code analyzers for both code and documentation. Once you're done with your changes, submit a pull request.
+Each pull request must pass all the RSpec specs, integration tests and meet our quality requirements.
 
-Coditsu will automatically check your work against our quality standards. You can find your commit check results on the [builds page](https://app.coditsu.io/karafka/repositories/waterdrop/builds/commit_builds) of WaterDrop repository.
+Fork it, update and wait for the Github Actions results.

data/certs/mensfeld.pem

@@ -1,25 +1,25 @@
 -----BEGIN CERTIFICATE-----
 MIIEODCCAqCgAwIBAgIBATANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhtYWNp
-ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMjAwODExMDkxNTM3WhcNMjEwODExMDkx
-NTM3WjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
-CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDCpXsCgmINb6lHBXXBdyrgsBPSxC4/
-2H+weJ6L9CruTiv2+2/ZkQGtnLcDgrD14rdLIHK7t0o3EKYlDT5GhD/XUVhI15JE
-N7IqnPUgexe1fbZArwQ51afxz2AmPQN2BkB2oeQHXxnSWUGMhvcEZpfbxCCJH26w
-hS0Ccsma8yxA6hSlGVhFVDuCr7c2L1di6cK2CtIDpfDaWqnVNJEwBYHIxrCoWK5g
-sIGekVt/admS9gRhIMaIBg+Mshth5/DEyWO2QjteTodItlxfTctrfmiAl8X8T5JP
-VXeLp5SSOJ5JXE80nShMJp3RFnGw5fqjX/ffjtISYh78/By4xF3a25HdWH9+qO2Z
-tx0wSGc9/4gqNM0APQnjN/4YXrGZ4IeSjtE+OrrX07l0TiyikzSLFOkZCAp8oBJi
-Fhlosz8xQDJf7mhNxOaZziqASzp/hJTU/tuDKl5+ql2icnMv5iV/i6SlmvU29QNg
-LCV71pUv0pWzN+OZbHZKWepGhEQ3cG9MwvkCAwEAAaN3MHUwCQYDVR0TBAIwADAL
-BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFImGed2AXS070ohfRidiCEhXEUN+MB0GA1Ud
+ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMjEwODExMTQxNTEzWhcNMjIwODExMTQx
+NTEzWjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
+CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDV2jKH4Ti87GM6nyT6D+ESzTI0MZDj
+ak2/TEwnxvijMJyCCPKT/qIkbW4/f0VHM4rhPr1nW73sb5SZBVFCLlJcOSKOBdUY
+TMY+SIXN2EtUaZuhAOe8LxtxjHTgRHvHcqUQMBENXTISNzCo32LnUxweu66ia4Pd
+1mNRhzOqNv9YiBZvtBf7IMQ+sYdOCjboq2dlsWmJiwiDpY9lQBTnWORnT3mQxU5x
+vPSwnLB854cHdCS8fQo4DjeJBRZHhEbcE5sqhEMB3RZA3EtFVEXOxlNxVTS3tncI
+qyNXiWDaxcipaens4ObSY1C2HTV7OWb7OMqSCIybeYTSfkaSdqmcl4S6zxXkjH1J
+tnjayAVzD+QVXGijsPLE2PFnJAh9iDET2cMsjabO1f6l1OQNyAtqpcyQcgfnyW0z
+g7tGxTYD+6wJHffM9d9txOUw6djkF6bDxyqB8lo4Z3IObCx18AZjI9XPS9QG7w6q
+LCWuMG2lkCcRgASqaVk9fEf9yMc2xxz5o3kCAwEAAaN3MHUwCQYDVR0TBAIwADAL
+BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFBqUFCKCOe5IuueUVqOB991jyCLLMB0GA1Ud
 EQQWMBSBEm1hY2llakBtZW5zZmVsZC5wbDAdBgNVHRIEFjAUgRJtYWNpZWpAbWVu
-c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBAKiHpwoENVrMi94V1zD4o8/6G3AU
-gWz4udkPYHTZLUy3dLznc/sNjdkJFWT3E6NKYq7c60EpJ0m0vAEg5+F5pmNOsvD3
-2pXLj9kisEeYhR516HwXAvtngboUcb75skqvBCU++4Pu7BRAPjO1/ihLSBexbwSS
-fF+J5OWNuyHHCQp+kGPLtXJe2yUYyvSWDj3I2//Vk0VhNOIlaCS1+5/P3ZJThOtm
-zJUBI7h3HgovwRpcnmk2mXTmU4Zx/bCzX8EA6VY0khEvnmiq7S6eBF0H9qH8KyQ6
-EkVLpvmUDFcf/uNaBQdazEMB5jYtwoA8gQlANETNGPi51KlkukhKgaIEDMkBDJOx
-65N7DzmkcyY0/GwjIVIxmRhcrCt1YeCUElmfFx0iida1/YRm6sB2AXqScc1+ECRi
-2DND//YJUikn1zwbz1kT70XmHd97B4Eytpln7K+M1u2g1pHVEPW4owD/ammXNpUy
-nt70FcDD4yxJQ+0YNiHd0N8IcVBM1TMIVctMNQ==
+c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBADD0/UuTTFgW+CGk2U0RDw2RBOca
+W2LTF/G7AOzuzD0Tc4voc7WXyrgKwJREv8rgBimLnNlgmFJLmtUCh2U/MgxvcilH
+yshYcbseNvjkrtYnLRlWZR4SSB6Zei5AlyGVQLPkvdsBpNegcG6w075YEwzX/38a
+8V9B/Yri2OGELBz8ykl7BsXUgNoUPA/4pHF6YRLz+VirOaUIQ4JfY7xGj6fSOWWz
+/rQ/d77r6o1mfJYM/3BRVg73a3b7DmRnE5qjwmSaSQ7u802pJnLesmArch0xGCT/
+fMmRli1Qb+6qOTl9mzD6UDMAyFR4t6MStLm0mIEqM0nBO5nUdUWbC7l9qXEf8XBE
+2DP28p3EqSuS+lKbAWKcqv7t0iRhhmaod+Yn9mcrLN1sa3q3KSQ9BCyxezCD4Mk2
+R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
+pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
 -----END CERTIFICATE-----

data/docker-compose.yml

@@ -5,7 +5,7 @@ services:
     ports:
       - "2181:2181"
   kafka:
-    image: wurstmeister/kafka:1.0.1
+    image: wurstmeister/kafka:2.12-2.5.0
     ports:
       - "9092:9092"
     environment:
@@ -13,5 +13,6 @@ services:
       KAFKA_ADVERTISED_PORT: 9092
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+      KAFKA_CREATE_TOPICS: 'example_topic:1:1'
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock

data/lib/water_drop/config.rb

@@ -7,33 +7,52 @@ module WaterDrop
   class Config
     include Dry::Configurable
 
+    # Defaults for kafka settings, that will be overwritten only if not present already
+    KAFKA_DEFAULTS = {
+      'client.id' => 'waterdrop'
+    }.freeze
+
+    private_constant :KAFKA_DEFAULTS
+
     # WaterDrop options
     #
     # option [String] id of the producer. This can be helpful when building producer specific
     #   instrumentation or loggers. It is not the kafka producer id
-    setting(:id, false) { |id| id || SecureRandom.uuid }
+    setting(
+      :id,
+      default: false,
+      constructor: ->(id) { id || SecureRandom.uuid }
+    )
     # option [Instance] logger that we want to use
     # @note Due to how rdkafka works, this setting is global for all the producers
-    setting(:logger, false) { |logger| logger || Logger.new($stdout, level: Logger::WARN) }
+    setting(
+      :logger,
+      default: false,
+      constructor: ->(logger) { logger || Logger.new($stdout, level: Logger::WARN) }
+    )
     # option [Instance] monitor that we want to use. See instrumentation part of the README for
     #   more details
-    setting(:monitor, false) { |monitor| monitor || WaterDrop::Instrumentation::Monitor.new }
+    setting(
+      :monitor,
+      default: false,
+      constructor: ->(monitor) { monitor || WaterDrop::Instrumentation::Monitor.new }
+    )
     # option [Integer] max payload size allowed for delivery to Kafka
-    setting :max_payload_size, 1_000_012
+    setting :max_payload_size, default: 1_000_012
     # option [Integer] Wait that long for the delivery report or raise an error if this takes
     #   longer than the timeout.
-    setting :max_wait_timeout, 5
+    setting :max_wait_timeout, default: 5
     # option [Numeric] how long should we wait between re-checks on the availability of the
     #   delivery report. In a really robust systems, this describes the min-delivery time
     #   for a single sync message when produced in isolation
-    setting :wait_timeout, 0.005 # 5 milliseconds
+    setting :wait_timeout, default: 0.005 # 5 milliseconds
     # option [Boolean] should we send messages. Setting this to false can be really useful when
     #   testing and or developing because when set to false, won't actually ping Kafka but will
     #   run all the validations, etc
-    setting :deliver, true
+    setting :deliver, default: true
     # rdkafka options
     # @see https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
-    setting :kafka, {}
+    setting :kafka, default: {}
 
     # Configuration method
     # @yield Runs a block of code providing a config singleton instance to it
@@ -41,12 +60,28 @@ module WaterDrop
     def setup
       configure do |config|
         yield(config)
+
+        merge_kafka_defaults!(config)
         validate!(config.to_h)
+
+        ::Rdkafka::Config.logger = config.logger
       end
     end
 
     private
 
+    # Propagates the kafka setting defaults unless they are already present
+    # This makes it easier to set some values that users usually don't change but still allows them
+    # to overwrite the whole hash if they want to
+    # @param config [Dry::Configurable::Config] dry config of this producer
+    def merge_kafka_defaults!(config)
+      KAFKA_DEFAULTS.each do |key, value|
+        next if config.kafka.key?(key)
+
+        config.kafka[key] = value
+      end
+    end
+
     # Validates the configuration and if anything is wrong, will raise an exception
     # @param config_hash [Hash] config hash with setup details
     # @raise [WaterDrop::Errors::ConfigurationInvalidError] raised when something is wrong with

data/lib/water_drop/contracts/message.rb

@@ -22,6 +22,7 @@ module WaterDrop
         required(:payload).filled(:str?)
         optional(:key).maybe(:str?, :filled?)
         optional(:partition).filled(:int?, gteq?: -1)
+        optional(:partition_key).maybe(:str?, :filled?)
         optional(:timestamp).maybe { time? | int? }
         optional(:headers).maybe(:hash?)
       end

data/lib/water_drop/instrumentation/callbacks/delivery.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Instrumentation
+    module Callbacks
+      # Creates a callable that we want to run upon each message delivery or failure
+      #
+      # @note We don't have to provide client_name here as this callback is per client instance
+      class Delivery
+        # @param producer_id [String] id of the current producer
+        # @param monitor [WaterDrop::Instrumentation::Monitor] monitor we are using
+        def initialize(producer_id, monitor)
+          @producer_id = producer_id
+          @monitor = monitor
+        end
+
+        # Emits delivery details to the monitor
+        # @param delivery_report [Rdkafka::Producer::DeliveryReport] delivery report
+        def call(delivery_report)
+          @monitor.instrument(
+            'message.acknowledged',
+            producer_id: @producer_id,
+            offset: delivery_report.offset,
+            partition: delivery_report.partition
+          )
+        end
+      end
+    end
+  end
+end