waterdrop 2.4.4 → 2.4.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e260fd0633bf1672fc6f84613d9cf47722d68112dbcc8575f1dca40a2628401b
-  data.tar.gz: 9528c011ffb175a22b34d59808dee41b735a008fca09f839b4a6418c2e198a2c
+  metadata.gz: 2a072e6e8ee65e0ce6f5c337c3a56b043140d0a5c60a1fd93d70b329c50932c5
+  data.tar.gz: 3b0d853ed72939da7302a89cca314d105bf3eef7c7abebe0c37c45960c2e8a80
 SHA512:
-  metadata.gz: d1021407b74f56a864cb06013c31303a5485b7ef85e1c7214374f679cc7a9c0a04a59e3600d56b1f36929af9870ff3997f4fac320a01e9c4a1256453142c6a8b
-  data.tar.gz: 2bcf613fa8be571699534cdd5fb08fe3665024cb2767096d2ba343f6965ec1a5c7db50bdb45fb078931b0eff2572e0176604b667af35d2c1f87d2ce8e508e583
+  metadata.gz: 50b61ccd6d84ee67a7de95dde8dfc926dd77cc0ebcba6299704a897da88c55a33352a87f0bed725160dd2859e6258d683dc73fec0c92bf3370eeddf1b4490e3c
+  data.tar.gz: a44bdb91da44a8767a385d829eaefd86f214c1baf03b5776c9d3f3d89ac6e2106b07b0da3185208b160110f0cfc5002bb779af66a4682db7d4d96211de7964fd

data/CHANGELOG.md

@@ -1,5 +1,28 @@
 # WaterDrop changelog
 
+## 2.4.6 (2022-12-10)
+- Set `statistics.interval.ms` to 5 seconds by default, so the defaults cover all the instrumentation out of the box.
+
+### Upgrade notes
+
+If you want to disable `librdkafka` statistics because you do not use them at all, update the `kafka` `statistics.interval.ms` setting and set it to `0`:
+
+```ruby
+producer = WaterDrop::Producer.new
+
+producer.setup do |config|
+  config.deliver = true
+  config.kafka = {
+    'bootstrap.servers': 'localhost:9092',
+    'statistics.interval.ms': 0
+  }
+end
+```
+
+## 2.4.5 (2022-12-10)
+- Fix invalid error scope visibility.
+- Cache partition count to improve messages production and lower stress on Kafka when `partition_key` is on.
+
 ## 2.4.4 (2022-12-09)
 - Add temporary patch on top of `rdkafka-ruby` to mitigate metadata fetch timeout failures.
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.4.4)
+    waterdrop (2.4.6)
       karafka-core (>= 2.0.6, < 3.0.0)
       zeitwerk (~> 2.3)
 

data/README.md

@@ -301,9 +301,9 @@ See the `WaterDrop::Instrumentation::Monitor::EVENTS` for the list of all the su
 
 ### Usage statistics
 
-WaterDrop may be configured to emit internal metrics at a fixed interval by setting the `kafka` `statistics.interval.ms` configuration property to a value > `0`. Once that is done, emitted statistics are available after subscribing to the `statistics.emitted` publisher event.
+WaterDrop is configured to emit internal `librdkafka` metrics every five seconds. You can change this by setting the `kafka` `statistics.interval.ms` configuration property to a value greater of equal `0`. Emitted statistics are available after subscribing to the `statistics.emitted` publisher event. If set to `0`, metrics will not be published.
 
-The statistics include all of the metrics from `librdkafka` (full list [here](https://github.com/edenhill/librdkafka/blob/master/STATISTICS.md)) as well as the diff of those against the previously emitted values.
+The statistics include all of the metrics from `librdkafka` (complete list [here](https://github.com/edenhill/librdkafka/blob/master/STATISTICS.md)) as well as the diff of those against the previously emitted values.
 
 For several attributes like `txmsgs`, `librdkafka` publishes only the totals. In order to make it easier to track the progress (for example number of messages sent between statistics emitted events), WaterDrop diffs all the numeric values against previously available numbers. All of those metrics are available under the same key as the metric but with additional `_d` postfix:
 

data/lib/waterdrop/config.rb

@@ -9,7 +9,10 @@ module WaterDrop
 
     # Defaults for kafka settings, that will be overwritten only if not present already
     KAFKA_DEFAULTS = {
-      'client.id': 'waterdrop'
+      'client.id': 'waterdrop',
+      # emit librdkafka statistics every five seconds. This is used in instrumentation.
+      # When disabled, part of metrics will not be published and available.
+      'statistics.interval.ms': 5_000
     }.freeze
 
     private_constant :KAFKA_DEFAULTS

data/lib/waterdrop/patches/rdkafka/metadata.rb

@@ -18,7 +18,7 @@ module WaterDrop
           attempt += 1
 
           super(*args)
-        rescue Rdkafka::RdkafkaError => e
+        rescue ::Rdkafka::RdkafkaError => e
           raise unless e.code == :timed_out
           raise if attempt > 10
 

data/lib/waterdrop/patches/rdkafka/producer.rb

@@ -7,6 +7,25 @@ module WaterDrop
     module Rdkafka
       # Rdkafka::Producer patches
       module Producer
+        # Cache partitions count for 30 seconds
+        PARTITIONS_COUNT_TTL = 30_000
+
+        private_constant :PARTITIONS_COUNT_TTL
+
+        # @param args [Object] arguments accepted by the original rdkafka producer
+        def initialize(*args)
+          super
+
+          @_partitions_count_cache = Concurrent::Hash.new do |cache, topic|
+            topic_metadata = ::Rdkafka::Metadata.new(inner_kafka, topic).topics&.first
+
+            cache[topic] = [
+              now,
+              topic_metadata ? topic_metadata[:partition_count] : nil
+            ]
+          end
+        end
+
         # Adds a method that allows us to get the native kafka producer name
         #
         # In between rdkafka versions, there are internal changes that force us to add some extra
@@ -14,23 +33,49 @@ module WaterDrop
         #
         # @return [String] producer instance name
         def name
-          unless @_native
+          @_name ||= ::Rdkafka::Bindings.rd_kafka_name(inner_kafka)
+        end
+
+        # This patch makes sure we cache the partition count for a given topic for given time
+        # This prevents us in case someone uses `partition_key` from querying for the count with
+        # each message. Instead we query once every 30 seconds at most
+        #
+        # @param topic [String] topic name
+        # @return [Integer] partition count for a given topic
+        def partition_count(topic)
+          closed_producer_check(__method__)
+
+          @_partitions_count_cache.delete_if do |_, cached|
+            now - cached.first > PARTITIONS_COUNT_TTL
+          end
+
+          @_partitions_count_cache[topic].last
+        end
+
+        # @return [FFI::Pointer] pointer to the raw librdkafka
+        def inner_kafka
+          unless @_inner_kafka
             version = ::Gem::Version.new(::Rdkafka::VERSION)
 
             if version < ::Gem::Version.new('0.12.0')
-              @native = @native_kafka
+              @_inner_kafka = @native_kafka
             elsif version < ::Gem::Version.new('0.13.0.beta.1')
-              @_native = @client.native
+              @_inner_kafka = @client.native
             else
-              @_native = @native_kafka.inner
+              @_inner_kafka = @native_kafka.inner
             end
           end
 
-          ::Rdkafka::Bindings.rd_kafka_name(@_native)
+          @_inner_kafka
+        end
+
+        # @return [Float] current clock time
+        def now
+          ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) * 1_000
         end
       end
     end
   end
 end
 
-::Rdkafka::Producer.include ::WaterDrop::Patches::Rdkafka::Producer
+::Rdkafka::Producer.prepend ::WaterDrop::Patches::Rdkafka::Producer

data/lib/waterdrop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '2.4.4'
+  VERSION = '2.4.6'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 2.4.4
+  version: 2.4.6
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2022-12-09 00:00:00.000000000 Z
+date: 2022-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core