waterdrop 2.4.7 → 2.4.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8aa78b8b5f2d8534689cb9fe3db46d579610ce3b4767cef46b16d8fb1e19d48e
-  data.tar.gz: 75ff38cc56317bc74047fe9ba53ee1f42b098299f694c1ad5767f80ed4c2cf7e
+  metadata.gz: e2e707af4409ace682680ca2131b89221ee04552b856a0855e663a209e09e0fb
+  data.tar.gz: fed47f90aa9397754ec6819fcc720d18ad3cace184a053ca6dbbec1b86480172
 SHA512:
-  metadata.gz: 3e5589c065d8db716a277bb78e985b85fbe94c2d064eff1cae780f9b4fa6f64ddd95ef0591c99de2dcefedd5582c80ad83442f1cd53091b27349fb193bcd98f6
-  data.tar.gz: 9af7835c4419dd10af2cde93b42c7797e1811f3f049fe7ae4a50a8ccad50fc6f86a8703777c60410d10c1615386eaee2acbeacf073dd640ba24af24ed534326a
+  metadata.gz: 8e2fa6d1abbaa0cae44a8a8fd5e17ef64a851e48c0433d74ca259a52747b92aa94beeabf2ccbc8c3df3efec67050f04b22cafb346f8f69d279effe86590a7079
+  data.tar.gz: 5eefb7ba3ae2c3f1ce6253fd5206762c4fd94698719968f42a3e7ddd5cc18a90459ed2eddc29c01452fd9c211cd1e57cb71f66338b428eaf71e0b3ddb0eb45c2

data/.github/workflows/ci.yml

@@ -16,11 +16,12 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
+          - '3.2'
           - '3.1'
           - '3.0'
           - '2.7'
         include:
-          - ruby: '3.1'
+          - ruby: '3.2'
             coverage: 'true'
     steps:
       - uses: actions/checkout@v3
@@ -62,7 +63,7 @@ jobs:
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 3.1
+          ruby-version: 3.2
       - name: Install latest bundler
         run: gem install bundler --no-document
       - name: Install Diffend plugin

data/.ruby-version

@@ -1 +1 @@
-3.1.3
+3.2.0

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # WaterDrop changelog
 
+## 2.4.9 (2023-01-11)
+- Remove empty debug logging out of `LoggerListener`.
+- Do not lock Ruby version in Karafka in favour of `karafka-core`.
+- Make sure `karafka-core` version is at least `2.0.9` to make sure we run `karafka-rdkafka`.
+
+## 2.4.8 (2023-01-07)
+- Use monotonic time from Karafka core.
+
 ## 2.4.7 (2022-12-18)
 - Add support to customizable middlewares that can modify message hash prior to validation and dispatch.
 - Fix a case where upon not-available leader, metadata request would not be retried

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.4.7)
-      karafka-core (>= 2.0.7, < 3.0.0)
+    waterdrop (2.4.9)
+      karafka-core (>= 2.0.9, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
@@ -22,30 +22,30 @@ GEM
     ffi (1.15.5)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.0.7)
+    karafka-core (2.0.9)
       concurrent-ruby (>= 1.1)
-      rdkafka (>= 0.12)
-    mini_portile2 (2.8.0)
-    minitest (5.16.3)
-    rake (13.0.6)
-    rdkafka (0.12.0)
+      karafka-rdkafka (>= 0.12)
+    karafka-rdkafka (0.12.0)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
+    mini_portile2 (2.8.1)
+    minitest (5.17.0)
+    rake (13.0.6)
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
       rspec-expectations (~> 3.12.0)
       rspec-mocks (~> 3.12.0)
     rspec-core (3.12.0)
       rspec-support (~> 3.12.0)
-    rspec-expectations (3.12.0)
+    rspec-expectations (3.12.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.0)
+    rspec-mocks (3.12.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
     rspec-support (3.12.0)
-    simplecov (0.21.2)
+    simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
@@ -56,7 +56,6 @@ GEM
     zeitwerk (2.6.6)
 
 PLATFORMS
-  arm64-darwin
   x86_64-linux
 
 DEPENDENCIES
@@ -67,4 +66,4 @@ DEPENDENCIES
   waterdrop!
 
 BUNDLED WITH
-   2.3.26
+   2.4.2

data/README.md

@@ -33,6 +33,7 @@ It:
 - [Instrumentation](#instrumentation)
   * [Usage statistics](#usage-statistics)
   * [Error notifications](#error-notifications)
+  * [Acknowledgment notifications](#acknowledgment-notifications)
   * [Datadog and StatsD integration](#datadog-and-statsd-integration)
   * [Forking and potential memory problems](#forking-and-potential-memory-problems)
 - [Middleware](#middleware)
@@ -352,6 +353,65 @@ producer.close
 
 Note: The metrics returned may not be completely consistent between brokers, toppars and totals, due to the internal asynchronous nature of librdkafka. E.g., the top level tx total may be less than the sum of the broker tx values which it represents.
 
+### Error notifications
+
+WaterDrop allows you to listen to all errors that occur while producing messages and in its internal background threads. Things like reconnecting to Kafka upon network errors and others unrelated to publishing messages are all available under `error.occurred` notification key. You can subscribe to this event to ensure your setup is healthy and without any problems that would otherwise go unnoticed as long as messages are delivered.
+
+```ruby
+producer = WaterDrop::Producer.new do |config|
+  # Note invalid connection port...
+  config.kafka = { 'bootstrap.servers': 'localhost:9090' }
+end
+
+producer.monitor.subscribe('error.occurred') do |event|
+  error = event[:error]
+
+  p "WaterDrop error occurred: #{error}"
+end
+
+# Run this code without Kafka cluster
+loop do
+  producer.produce_async(topic: 'events', payload: 'data')
+
+  sleep(1)
+end
+
+# After you stop your Kafka cluster, you will see a lot of those:
+#
+# WaterDrop error occurred: Local: Broker transport failure (transport)
+#
+# WaterDrop error occurred: Local: Broker transport failure (transport)
+```
+
+### Acknowledgment notifications
+
+WaterDrop allows you to listen to Kafka messages' acknowledgment events. This will enable you to monitor deliveries of messages from WaterDrop even when using asynchronous dispatch methods.
+
+That way, you can make sure, that dispatched messages are acknowledged by Kafka.
+
+```ruby
+producer = WaterDrop::Producer.new do |config|
+  config.kafka = { 'bootstrap.servers': 'localhost:9092' }
+end
+
+producer.monitor.subscribe('message.acknowledged') do |event|
+  producer_id = event[:producer_id]
+  offset = event[:offset]
+
+  p "WaterDrop [#{producer_id}] delivered message with offset: #{offset}"
+end
+
+loop do
+  producer.produce_async(topic: 'events', payload: 'data')
+
+  sleep(1)
+end
+
+# WaterDrop [dd8236fff672] delivered message with offset: 32
+# WaterDrop [dd8236fff672] delivered message with offset: 33
+# WaterDrop [dd8236fff672] delivered message with offset: 34
+```
+
 ### Datadog and StatsD integration
 
 WaterDrop comes with (optional) full Datadog and StatsD integration that you can use. To use it:
@@ -385,36 +445,6 @@ You can also find [here](https://github.com/karafka/waterdrop/blob/master/lib/wa
 
 ![Example WaterDrop DD dashboard](https://raw.githubusercontent.com/karafka/misc/master/printscreens/waterdrop_dd_dashboard_example.png)
 
-### Error notifications
-
-WaterDrop allows you to listen to all errors that occur while producing messages and in its internal background threads. Things like reconnecting to Kafka upon network errors and others unrelated to publishing messages are all available under `error.occurred` notification key. You can subscribe to this event to ensure your setup is healthy and without any problems that would otherwise go unnoticed as long as messages are delivered.
-
-```ruby
-producer = WaterDrop::Producer.new do |config|
-  # Note invalid connection port...
-  config.kafka = { 'bootstrap.servers': 'localhost:9090' }
-end
-
-producer.monitor.subscribe('error.occurred') do |event|
-  error = event[:error]
-
-  p "WaterDrop error occurred: #{error}"
-end
-
-# Run this code without Kafka cluster
-loop do
-  producer.produce_async(topic: 'events', payload: 'data')
-
-  sleep(1)
-end
-
-# After you stop your Kafka cluster, you will see a lot of those:
-#
-# WaterDrop error occurred: Local: Broker transport failure (transport)
-#
-# WaterDrop error occurred: Local: Broker transport failure (transport)
-```
-
 ### Forking and potential memory problems
 
 If you work with forked processes, make sure you **don't** use the producer before the fork. You can easily configure the producer and then fork and use it.

data/lib/waterdrop/contracts/config.rb

@@ -7,7 +7,7 @@ module WaterDrop
       configure do |config|
         config.error_messages = YAML.safe_load(
           File.read(
-            File.join(WaterDrop.gem_root, 'config', 'errors.yml')
+            File.join(WaterDrop.gem_root, 'config', 'locales', 'errors.yml')
           )
         ).fetch('en').fetch('validations').fetch('config')
       end

data/lib/waterdrop/contracts/message.rb

@@ -8,7 +8,7 @@ module WaterDrop
       configure do |config|
         config.error_messages = YAML.safe_load(
           File.read(
-            File.join(WaterDrop.gem_root, 'config', 'errors.yml')
+            File.join(WaterDrop.gem_root, 'config', 'locales', 'errors.yml')
           )
         ).fetch('en').fetch('validations').fetch('message')
       end

data/lib/waterdrop/instrumentation/logger_listener.rb

@@ -82,7 +82,6 @@ module WaterDrop
       # @param event [Dry::Events::Event] event that happened with the details
       def on_producer_closed(event)
         info event, 'Closing producer'
-        debug event, ''
       end
 
       # @param event [Dry::Events::Event] event that happened with the error details
@@ -91,7 +90,6 @@ module WaterDrop
         type = event[:type]
 
         error(event, "Error occurred: #{error} - #{type}")
-        debug(event, '')
       end
 
       private

data/lib/waterdrop/patches/rdkafka/producer.rb

@@ -7,6 +7,8 @@ module WaterDrop
     module Rdkafka
       # Rdkafka::Producer patches
       module Producer
+        include ::Karafka::Core::Helpers::Time
+
         # Cache partitions count for 30 seconds
         PARTITIONS_COUNT_TTL = 30_000
 
@@ -20,7 +22,7 @@ module WaterDrop
             topic_metadata = ::Rdkafka::Metadata.new(inner_kafka, topic).topics&.first
 
             cache[topic] = [
-              now,
+              monotonic_now,
               topic_metadata ? topic_metadata[:partition_count] : nil
             ]
           end
@@ -46,7 +48,7 @@ module WaterDrop
           closed_producer_check(__method__)
 
           @_partitions_count_cache.delete_if do |_, cached|
-            now - cached.first > PARTITIONS_COUNT_TTL
+            monotonic_now - cached.first > PARTITIONS_COUNT_TTL
           end
 
           @_partitions_count_cache[topic].last
@@ -68,11 +70,6 @@ module WaterDrop
 
           @_inner_kafka
         end
-
-        # @return [Float] current clock time
-        def now
-          ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) * 1_000
-        end
       end
     end
   end

data/lib/waterdrop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '2.4.7'
+  VERSION = '2.4.9'
 end

data/waterdrop.gemspec

@@ -16,11 +16,9 @@ Gem::Specification.new do |spec|
   spec.description   = spec.summary
   spec.license       = 'MIT'
 
-  spec.add_dependency 'karafka-core', '>= 2.0.7', '< 3.0.0'
+  spec.add_dependency 'karafka-core', '>= 2.0.9', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
-  spec.required_ruby_version = '>= 2.7'
-
   if $PROGRAM_NAME.end_with?('gem')
     spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 2.4.7
+  version: 2.4.9
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2022-12-18 00:00:00.000000000 Z
+date: 2023-01-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.7
+        version: 2.0.9
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -53,7 +53,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.7
+        version: 2.0.9
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -92,7 +92,7 @@ files:
 - MIT-LICENSE
 - README.md
 - certs/cert_chain.pem
-- config/errors.yml
+- config/locales/errors.yml
 - docker-compose.yml
 - lib/waterdrop.rb
 - lib/waterdrop/config.rb
@@ -135,14 +135,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.7'
+      version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
+rubygems_version: 3.3.4
 signing_key: 
 specification_version: 4
 summary: Kafka messaging made easy!