waterdrop 2.4.3 → 2.4.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 065c53c0e711cfe49e718a321ce64740b68a45bd6ac477c9eaf2a79876de8df7
-  data.tar.gz: e77179ca5162c4862369838197a62063b5127b4d3a72b7ab08f85f930cee6ea6
+  metadata.gz: e1cabe8225d1038bc9fab5c41efaf5822b3d6a633dac233f92058874be8cb5ae
+  data.tar.gz: ac830393a2c6ed2d50042a08fbd59a6d2ebbd0de505a53953c67b3d50ac1dd22
 SHA512:
-  metadata.gz: 76af1a5f87a5f31bda8b9f5f111173e1992f6d5ddc85fa2b43d5e1fbd45b93e67601691cbf13b3b2a2e8eb49bc04196b84c5eb81c5b0bd6e11bbe014220fe821
-  data.tar.gz: 27c667d0c93f6948a85d9411fd1dfa46bd38ebb36f7c6907785658bb58ae3001b8d2b81bcb1a9faac64acc5248bc1300a3530736cdd7193b155e01eca4eefbbe
+  metadata.gz: d757751cb8a62fee91c5aff7ac30260e3b45d62d17db71eedcd23b77881b4564b8dc0bef534eb060049a65b5d99610c09c86eeb13913a999d864dffc3adee5e3
+  data.tar.gz: 51cb53cc3a4f168b3157ad840157f2df8a9479381972fe0d8cc13b445449099373acd76d37cf0206ec35d625fb75723b3ab4a3e0c193413de9844c54106209c5

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # WaterDrop changelog
 
+## 2.4.5 (2022-12-10)
+- Fix invalid error scope visibility.
+- Cache partition count to improve messages production and lower stress on Kafka when `partition_key` is on.
+
+## 2.4.4 (2022-12-09)
+- Add temporary patch on top of `rdkafka-ruby` to mitigate metadata fetch timeout failures.
+
 ## 2.4.3 (2022-12-07)
 - Support for librdkafka 0.13
 - Update Github Actions

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.4.3)
+    waterdrop (2.4.5)
       karafka-core (>= 2.0.6, < 3.0.0)
       zeitwerk (~> 2.3)
 

data/lib/waterdrop/patches/rdkafka/metadata.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  # Patches to external components
+  module Patches
+    # Rdkafka related patches
+    module Rdkafka
+      # Rdkafka::Metadata patches
+      module Metadata
+        # We overwrite this method because there were reports of metadata operation timing out
+        # when Kafka was under stress. While the messages dispatch will be retried, metadata
+        # fetch happens prior to that, effectively crashing the process. Metadata fetch was not
+        # being retried at all.
+        #
+        # @param args [Array<Object>] all the metadata original arguments
+        def initialize(*args)
+          attempt ||= 0
+          attempt += 1
+
+          super(*args)
+        rescue ::Rdkafka::RdkafkaError => e
+          raise unless e.code == :timed_out
+          raise if attempt > 10
+
+          backoff_factor = 2**attempt
+          timeout = backoff_factor * 0.1
+
+          sleep(timeout)
+
+          retry
+        end
+      end
+    end
+  end
+end
+
+::Rdkafka::Metadata.prepend ::WaterDrop::Patches::Rdkafka::Metadata

data/lib/waterdrop/patches/rdkafka/producer.rb

@@ -7,6 +7,25 @@ module WaterDrop
     module Rdkafka
       # Rdkafka::Producer patches
       module Producer
+        # Cache partitions count for 30 seconds
+        PARTITIONS_COUNT_TTL = 30_000
+
+        private_constant :PARTITIONS_COUNT_TTL
+
+        # @param args [Object] arguments accepted by the original rdkafka producer
+        def initialize(*args)
+          super
+
+          @_partitions_count_cache = Concurrent::Hash.new do |cache, topic|
+            topic_metadata = ::Rdkafka::Metadata.new(inner_kafka, topic).topics&.first
+
+            cache[topic] = [
+              now,
+              topic_metadata ? topic_metadata[:partition_count] : nil
+            ]
+          end
+        end
+
         # Adds a method that allows us to get the native kafka producer name
         #
         # In between rdkafka versions, there are internal changes that force us to add some extra
@@ -14,23 +33,49 @@ module WaterDrop
         #
         # @return [String] producer instance name
         def name
-          unless @_native
+          @_name ||= ::Rdkafka::Bindings.rd_kafka_name(inner_kafka)
+        end
+
+        # This patch makes sure we cache the partition count for a given topic for given time
+        # This prevents us in case someone uses `partition_key` from querying for the count with
+        # each message. Instead we query once every 30 seconds at most
+        #
+        # @param topic [String] topic name
+        # @return [Integer] partition count for a given topic
+        def partition_count(topic)
+          closed_producer_check(__method__)
+
+          @_partitions_count_cache.delete_if do |_, cached|
+            now - cached.first > PARTITIONS_COUNT_TTL
+          end
+
+          @_partitions_count_cache[topic].last
+        end
+
+        # @return [FFI::Pointer] pointer to the raw librdkafka
+        def inner_kafka
+          unless @_inner_kafka
             version = ::Gem::Version.new(::Rdkafka::VERSION)
 
             if version < ::Gem::Version.new('0.12.0')
-              @native = @native_kafka
+              @_inner_kafka = @native_kafka
             elsif version < ::Gem::Version.new('0.13.0.beta.1')
-              @_native = @client.native
+              @_inner_kafka = @client.native
             else
-              @_native = @native_kafka.inner
+              @_inner_kafka = @native_kafka.inner
             end
           end
 
-          ::Rdkafka::Bindings.rd_kafka_name(@_native)
+          @_inner_kafka
+        end
+
+        # @return [Float] current clock time
+        def now
+          ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) * 1_000
         end
       end
     end
   end
 end
 
-::Rdkafka::Producer.include ::WaterDrop::Patches::Rdkafka::Producer
+::Rdkafka::Producer.prepend ::WaterDrop::Patches::Rdkafka::Producer

data/lib/waterdrop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '2.4.3'
+  VERSION = '2.4.5'
 end

data.tar.gz.sig

@@ -1 +1,5 @@
-�V�i�ڈg� �J�Ł�J�})g�������.�J�Q�Sn̡��؊��@lG_�t&A���<e{���|���=@;��@��p�x[(M��8��_+|���0[��$���g���t #XIw?�=X�%Ԉ�qoA�:��}��a�T���`�	��K�
+w�9T���BS�C��=Y<_ʀ����:����-+A
�Y���Lo��.}�VNyҙH.�m�Ju�4����؄\Y2��9�s�]Ї�AԷ[�7�JQ�� 3��]ʳ���Su}"
����samB봧�^S����n���K��b�N��F�Ԙ����g��m�,�4r��M�B` �
+�H>��Kh�*���7��
+�nA�*�%gOn�z����!CWAP
!SҼ8h��n���9z�'�F��j�òr@D�>�V�~�
+Rj,������и]��cGv���U���2�������L�
+�r~�3���S`���۹�?�

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 2.4.3
+  version: 2.4.5
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2022-12-07 00:00:00.000000000 Z
+date: 2022-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -108,6 +108,7 @@ files:
 - lib/waterdrop/instrumentation/notifications.rb
 - lib/waterdrop/instrumentation/vendors/datadog/dashboard.json
 - lib/waterdrop/instrumentation/vendors/datadog/listener.rb
+- lib/waterdrop/patches/rdkafka/metadata.rb
 - lib/waterdrop/patches/rdkafka/producer.rb
 - lib/waterdrop/producer.rb
 - lib/waterdrop/producer/async.rb