waterdrop 2.4.4 → 2.4.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e260fd0633bf1672fc6f84613d9cf47722d68112dbcc8575f1dca40a2628401b
-  data.tar.gz: 9528c011ffb175a22b34d59808dee41b735a008fca09f839b4a6418c2e198a2c
+  metadata.gz: e1cabe8225d1038bc9fab5c41efaf5822b3d6a633dac233f92058874be8cb5ae
+  data.tar.gz: ac830393a2c6ed2d50042a08fbd59a6d2ebbd0de505a53953c67b3d50ac1dd22
 SHA512:
-  metadata.gz: d1021407b74f56a864cb06013c31303a5485b7ef85e1c7214374f679cc7a9c0a04a59e3600d56b1f36929af9870ff3997f4fac320a01e9c4a1256453142c6a8b
-  data.tar.gz: 2bcf613fa8be571699534cdd5fb08fe3665024cb2767096d2ba343f6965ec1a5c7db50bdb45fb078931b0eff2572e0176604b667af35d2c1f87d2ce8e508e583
+  metadata.gz: d757751cb8a62fee91c5aff7ac30260e3b45d62d17db71eedcd23b77881b4564b8dc0bef534eb060049a65b5d99610c09c86eeb13913a999d864dffc3adee5e3
+  data.tar.gz: 51cb53cc3a4f168b3157ad840157f2df8a9479381972fe0d8cc13b445449099373acd76d37cf0206ec35d625fb75723b3ab4a3e0c193413de9844c54106209c5

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # WaterDrop changelog
 
+## 2.4.5 (2022-12-10)
+- Fix invalid error scope visibility.
+- Cache partition count to improve messages production and lower stress on Kafka when `partition_key` is on.
+
 ## 2.4.4 (2022-12-09)
 - Add temporary patch on top of `rdkafka-ruby` to mitigate metadata fetch timeout failures.
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.4.4)
+    waterdrop (2.4.5)
       karafka-core (>= 2.0.6, < 3.0.0)
       zeitwerk (~> 2.3)
 

data/lib/waterdrop/patches/rdkafka/metadata.rb

@@ -18,7 +18,7 @@ module WaterDrop
           attempt += 1
 
           super(*args)
-        rescue Rdkafka::RdkafkaError => e
+        rescue ::Rdkafka::RdkafkaError => e
           raise unless e.code == :timed_out
           raise if attempt > 10
 

data/lib/waterdrop/patches/rdkafka/producer.rb

@@ -7,6 +7,25 @@ module WaterDrop
     module Rdkafka
       # Rdkafka::Producer patches
       module Producer
+        # Cache partitions count for 30 seconds
+        PARTITIONS_COUNT_TTL = 30_000
+
+        private_constant :PARTITIONS_COUNT_TTL
+
+        # @param args [Object] arguments accepted by the original rdkafka producer
+        def initialize(*args)
+          super
+
+          @_partitions_count_cache = Concurrent::Hash.new do |cache, topic|
+            topic_metadata = ::Rdkafka::Metadata.new(inner_kafka, topic).topics&.first
+
+            cache[topic] = [
+              now,
+              topic_metadata ? topic_metadata[:partition_count] : nil
+            ]
+          end
+        end
+
         # Adds a method that allows us to get the native kafka producer name
         #
         # In between rdkafka versions, there are internal changes that force us to add some extra
@@ -14,23 +33,49 @@ module WaterDrop
         #
         # @return [String] producer instance name
         def name
-          unless @_native
+          @_name ||= ::Rdkafka::Bindings.rd_kafka_name(inner_kafka)
+        end
+
+        # This patch makes sure we cache the partition count for a given topic for given time
+        # This prevents us in case someone uses `partition_key` from querying for the count with
+        # each message. Instead we query once every 30 seconds at most
+        #
+        # @param topic [String] topic name
+        # @return [Integer] partition count for a given topic
+        def partition_count(topic)
+          closed_producer_check(__method__)
+
+          @_partitions_count_cache.delete_if do |_, cached|
+            now - cached.first > PARTITIONS_COUNT_TTL
+          end
+
+          @_partitions_count_cache[topic].last
+        end
+
+        # @return [FFI::Pointer] pointer to the raw librdkafka
+        def inner_kafka
+          unless @_inner_kafka
             version = ::Gem::Version.new(::Rdkafka::VERSION)
 
             if version < ::Gem::Version.new('0.12.0')
-              @native = @native_kafka
+              @_inner_kafka = @native_kafka
             elsif version < ::Gem::Version.new('0.13.0.beta.1')
-              @_native = @client.native
+              @_inner_kafka = @client.native
             else
-              @_native = @native_kafka.inner
+              @_inner_kafka = @native_kafka.inner
             end
           end
 
-          ::Rdkafka::Bindings.rd_kafka_name(@_native)
+          @_inner_kafka
+        end
+
+        # @return [Float] current clock time
+        def now
+          ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) * 1_000
         end
       end
     end
   end
 end
 
-::Rdkafka::Producer.include ::WaterDrop::Patches::Rdkafka::Producer
+::Rdkafka::Producer.prepend ::WaterDrop::Patches::Rdkafka::Producer

data/lib/waterdrop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '2.4.4'
+  VERSION = '2.4.5'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 2.4.4
+  version: 2.4.5
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2022-12-09 00:00:00.000000000 Z
+date: 2022-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core