waterdrop 2.5.3 → 2.6.1.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ca3df6bcca5db1eed27ca612023b0df892e9a4c790b28353180184e0bdd4d784
-  data.tar.gz: f4d60c59bbfff8a8af5d64f7580b4fa0509e15b3f8382849913a80a2e49830e5
+  metadata.gz: 5e637029ce9d9113e2d64cadcec57ac980602d489f2207507194082065604b16
+  data.tar.gz: 21d0a5a643b3e48603fe8e8adb13dbe827a0b9b50871675e57aa62b71abe7573
 SHA512:
-  metadata.gz: 60d191c91cf40275895324a17c5f82838fe2f5215b3da983ad1008fa6a2ebf30ac228451b6108af5db56b0f104578541a6e35dc9fb645d0e74e38f7f9e99f233
-  data.tar.gz: 7185c404cdcfb406ee19be95290d85a316291268bdc5c668fb482c4f496d55e648f4604b5a707cbb1d6ed6b9d79553753f6e123714c19932e9088e83b3cf5f58
+  metadata.gz: e34c6c5225022f191dd80da877126b5c37075b3356b1e99e4e6b79adb0c54b8939e6b43f2efded8f68a0b68424b3cdbdcaa8abe19a781456faa4f653ae9063ea
+  data.tar.gz: 5060d94c549d5d0ac4b2e09f8ab4888643008ea732943e76f75dfedd386f08ada8a60ad278f85dcfa1172bf813c993a38b2feb71ee75ea2ce1a72bf3021edbcb

data/CHANGELOG.md

@@ -1,9 +1,29 @@
 # WaterDrop changelog
 
+### 2.6.1 (Unreleased)
+- [Refactor] Remove no longer needed patches.
+- [Fix] Fork detection on a short lived processes seems to fail. Clear the used parent process client reference not to close it in the finalizer (#356).
+- [Change] Require `karafka-rdkafka` `>= 0.13.0.beta2`.
+- [Change] Require 'karafka-core' `>= 2.1.0`
+
+### 2.6.0 (2023-06-11)
+- [Improvement] Introduce `client_class` setting for ability to replace underlying client with anything specific to a given env (dev, test, etc).
+- [Improvement] Introduce `Clients::Buffered` useful for writing specs that do not have to talk with Kafka (id-ilych)
+- [Improvement] Make `#produce` method private to avoid confusion and make sure it is not used directly (it is not part of the official API).
+- [Change] Change `wait_on_queue_full` from `false` to `true` as a default.
+- [Change] Rename `wait_on_queue_full_timeout` to `wait_backoff_on_queue_full` to match what it actually does.
+- [Enhancement] Introduce `wait_timeout_on_queue_full` with proper meaning. That is, this represents time after which despite backoff the error will be raised. This should allow to raise an error in case the backoff attempts were insufficient. This prevents from a case, where upon never deliverable messages we would end up with an infinite loop.
+- [Fix] Provide `type` for queue full errors that references the appropriate public API method correctly.
+
+### Upgrade notes
+
+1. Rename `wait_on_queue_full_timeout` to `wait_backoff_on_queue_full`.
+2. Set `wait_on_queue_full` to `false` if you did not use it and do not want.
+
 ## 2.5.3 (2023-05-26)
-- Require `karafka-core` `2.0.13`
-- Include topic name in the `error.occurred` notification payload.
-- Include topic name in the `message.acknowledged` notification payload.
+- [Enhancement] Include topic name in the `error.occurred` notification payload.
+- [Enhancement] Include topic name in the `message.acknowledged` notification payload.
+- [Maintenance] Require `karafka-core` `2.0.13`
 
 ## 2.5.2 (2023-04-24)
 - [Fix] Require missing Pathname (#345)

data/Gemfile.lock

@@ -1,14 +1,14 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.5.3)
-      karafka-core (>= 2.0.13, < 3.0.0)
+    waterdrop (2.6.1.beta1)
+      karafka-core (>= 2.1.0.beta1, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (7.0.4.3)
+    activesupport (7.0.5)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -20,25 +20,25 @@ GEM
     factory_bot (6.2.1)
       activesupport (>= 5.0.0)
     ffi (1.15.5)
-    i18n (1.12.0)
+    i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.0.13)
+    karafka-core (2.1.0.beta1)
       concurrent-ruby (>= 1.1)
-      karafka-rdkafka (>= 0.12.3)
-    karafka-rdkafka (0.12.3)
+      karafka-rdkafka (>= 0.13.0.beta2, < 0.14.0)
+    karafka-rdkafka (0.13.0.beta2)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    mini_portile2 (2.8.1)
+    mini_portile2 (2.8.2)
     minitest (5.18.0)
     rake (13.0.6)
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
       rspec-expectations (~> 3.12.0)
       rspec-mocks (~> 3.12.0)
-    rspec-core (3.12.1)
+    rspec-core (3.12.2)
       rspec-support (~> 3.12.0)
-    rspec-expectations (3.12.2)
+    rspec-expectations (3.12.3)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
     rspec-mocks (3.12.5)
@@ -53,10 +53,9 @@ GEM
     simplecov_json_formatter (0.1.4)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    zeitwerk (2.6.7)
+    zeitwerk (2.6.8)
 
 PLATFORMS
-  arm64-darwin-21
   x86_64-linux
 
 DEPENDENCIES

data/README.md

@@ -93,15 +93,16 @@ end
 
 Some of the options are:
 
-| Option                       | Description                                                      |
-|------------------------------|------------------------------------------------------------------|
-| `id`                         | id of the producer for instrumentation and logging               |
-| `logger`                     | Logger that we want to use                                       |
-| `deliver`                    | Should we send messages to Kafka or just fake the delivery       |
-| `max_wait_timeout`           | Waits that long for the delivery report or raises an error       |
-| `wait_timeout`               | Waits that long before re-check of delivery report availability  |
-| `wait_on_queue_full`         | Should be wait on queue full or raise an error when that happens |
-| `wait_on_queue_full_timeout` | Waits that long before retry when queue is full                  |
+| Option                       | Description                                                                |
+|------------------------------|----------------------------------------------------------------------------|
+| `id`                         | id of the producer for instrumentation and logging                         |
+| `logger`                     | Logger that we want to use                                                 |
+| `deliver`                    | Should we send messages to Kafka or just fake the delivery                 |
+| `max_wait_timeout`           | Waits that long for the delivery report or raises an error                 |
+| `wait_timeout`               | Waits that long before re-check of delivery report availability            |
+| `wait_on_queue_full`         | Should be wait on queue full or raise an error when that happens           |
+| `wait_backoff_on_queue_full` | Waits that long before retry when queue is full                            |
+| `wait_timeout_on_queue_full` | If back-offs and attempts that that much time, error won't be retried more |
 
 Full list of the root configuration options is available [here](https://github.com/karafka/waterdrop/blob/master/lib/waterdrop/config.rb#L25).
 
@@ -332,6 +333,12 @@ producer.close
 
 See the `WaterDrop::Instrumentation::Notifications::EVENTS` for the list of all the supported events.
 
+### Karafka Web-UI
+
+Karafka [Web UI](https://karafka.io/docs/Web-UI-Getting-Started/) is a user interface for the Karafka framework. The Web UI provides a convenient way for monitor all producers related errors out of the box.
+
+![Example producer errors in Karafka Web-UI](https://raw.githubusercontent.com/karafka/misc/master/printscreens/web-ui/errors-producer.png)
+
 ### Usage statistics
 
 WaterDrop is configured to emit internal `librdkafka` metrics every five seconds. You can change this by setting the `kafka` `statistics.interval.ms` configuration property to a value greater of equal `0`. Emitted statistics are available after subscribing to the `statistics.emitted` publisher event. If set to `0`, metrics will not be published.

data/config/locales/errors.yml

@@ -10,6 +10,9 @@ en:
       max_wait_timeout_format: must be an integer that is equal or bigger than 0
       kafka_format: must be a hash with symbol based keys
       kafka_key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
+      wait_on_queue_full_format: must be boolean
+      wait_backoff_on_queue_full_format: must be a numeric that is bigger or equal to 0
+      wait_timeout_on_queue_full_format: must be a numeric that is bigger or equal to 0
 
     message:
       missing: must be present

data/lib/waterdrop/clients/buffered.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Clients
+    # Client used to buffer messages that we send out in specs and other places.
+    class Buffered < Clients::Dummy
+      attr_accessor :messages
+
+      # Sync fake response for the message delivery to Kafka, since we do not dispatch anything
+      class SyncResponse
+        # @param _args Handler wait arguments (irrelevant as waiting is fake here)
+        def wait(*_args)
+          false
+        end
+      end
+
+      # @param args [Object] anything accepted by `Clients::Dummy`
+      def initialize(*args)
+        super
+        @messages = []
+        @topics = Hash.new { |k, v| k[v] = [] }
+      end
+
+      # "Produces" message to Kafka: it acknowledges it locally, adds it to the internal buffer
+      # @param message [Hash] `WaterDrop::Producer#produce_sync` message hash
+      def produce(message)
+        topic = message.fetch(:topic) { raise ArgumentError, ':topic is missing' }
+        @topics[topic] << message
+        @messages << message
+        SyncResponse.new
+      end
+
+      # Returns messages produced to a given topic
+      # @param topic [String]
+      def messages_for(topic)
+        @topics[topic]
+      end
+
+      # Clears internal buffer
+      # Used in between specs so messages do not leak out
+      def reset
+        @messages.clear
+        @topics.each_value(&:clear)
+      end
+    end
+  end
+end

data/lib/waterdrop/{producer/dummy_client.rb → clients/dummy.rb}

@@ -1,12 +1,15 @@
 # frozen_string_literal: true
 
 module WaterDrop
-  class Producer
+  module Clients
     # A dummy client that is supposed to be used instead of Rdkafka::Producer in case we don't
-    # want to dispatch anything to Kafka
-    class DummyClient
-      # @return [DummyClient] dummy instance
-      def initialize
+    # want to dispatch anything to Kafka.
+    #
+    # It does not store anything and just ignores messages.
+    class Dummy
+      # @param _producer [WaterDrop::Producer]
+      # @return [Dummy] dummy instance
+      def initialize(_producer)
         @counter = -1
       end
 

data/lib/waterdrop/clients/rdkafka.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  # Namespace for all the clients that WaterDrop may use under the hood
+  module Clients
+    # Default Rdkafka client.
+    # Since we use the ::Rdkafka::Producer under the hood, this is just a module that aligns with
+    # client building API for the convenience.
+    module Rdkafka
+      class << self
+        # @param producer [WaterDrop::Producer] producer instance with its config, etc
+        # @note We overwrite this that way, because we do not care
+        def new(producer)
+          client = ::Rdkafka::Config.new(producer.config.kafka.to_h).producer
+
+          # This callback is not global and is per client, thus we do not have to wrap it with a
+          # callbacks manager to make it work
+          client.delivery_callback = Instrumentation::Callbacks::Delivery.new(
+            producer.id,
+            producer.config.monitor
+          )
+
+          client
+        end
+      end
+    end
+  end
+end

data/lib/waterdrop/config.rb

@@ -56,15 +56,20 @@ module WaterDrop
     #   in the `error.occurred` notification pipeline with a proper type as while this is
     #   recoverable, in a high number it still may mean issues.
     #   Waiting is one of the recommended strategies.
-    setting :wait_on_queue_full, default: false
+    setting :wait_on_queue_full, default: true
     # option [Integer] how long (in seconds) should we backoff before a retry when queue is full
     #   The retry will happen with the same message and backoff should give us some time to
     #   dispatch previously buffered messages.
-    setting :wait_on_queue_full_timeout, default: 0.1
+    setting :wait_backoff_on_queue_full, default: 0.1
+    # option [Numeric] how many seconds should we wait with the backoff on queue having space for
+    # more messages before re-raising the error.
+    setting :wait_timeout_on_queue_full, default: 10
     # option [Boolean] should we send messages. Setting this to false can be really useful when
     #   testing and or developing because when set to false, won't actually ping Kafka but will
     #   run all the validations, etc
     setting :deliver, default: true
+    # option [Class] class for usage when creating the underlying client used to dispatch messages
+    setting :client_class, default: Clients::Rdkafka
     # rdkafka options
     # @see https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
     setting :kafka, default: {}

data/lib/waterdrop/contracts/config.rb

@@ -19,6 +19,9 @@ module WaterDrop
       required(:max_wait_timeout) { |val| val.is_a?(Numeric) && val >= 0 }
       required(:wait_timeout) { |val| val.is_a?(Numeric) && val.positive? }
       required(:kafka) { |val| val.is_a?(Hash) && !val.empty? }
+      required(:wait_on_queue_full) { |val| [true, false].include?(val) }
+      required(:wait_backoff_on_queue_full) { |val| val.is_a?(Numeric) && val >= 0 }
+      required(:wait_timeout_on_queue_full) { |val| val.is_a?(Numeric) && val >= 0 }
 
       # rdkafka allows both symbols and strings as keys for config but then casts them to strings
       # This can be confusing, so we expect all keys to be symbolized

data/lib/waterdrop/producer/builder.rb

@@ -10,18 +10,13 @@ module WaterDrop
       # @return [Rdkafka::Producer, Producer::DummyClient] raw rdkafka producer or a dummy producer
       #   when we don't want to dispatch any messages
       def call(producer, config)
-        return DummyClient.new unless config.deliver
+        klass = config.client_class
+        # This allows us to have backwards compatibility.
+        # If it is the default client and delivery is set to false, we use dummy as we used to
+        # before `client_class` was introduced
+        klass = Clients::Dummy if klass == Clients::Rdkafka && !config.deliver
 
-        client = Rdkafka::Config.new(config.kafka.to_h).producer
-
-        # This callback is not global and is per client, thus we do not have to wrap it with a
-        # callbacks manager to make it work
-        client.delivery_callback = Instrumentation::Callbacks::Delivery.new(
-          producer.id,
-          config.monitor
-        )
-
-        client
+        klass.new(producer)
       end
     end
   end

data/lib/waterdrop/producer.rb

@@ -7,6 +7,7 @@ module WaterDrop
     include Sync
     include Async
     include Buffer
+    include ::Karafka::Core::Helpers::Time
 
     # Which of the inline flow errors do we want to intercept and re-bind
     SUPPORTED_FLOW_ERRORS = [
@@ -75,13 +76,18 @@ module WaterDrop
       @connecting_mutex.synchronize do
         return @client if @client && @pid == Process.pid
 
-        # We should raise an error when trying to use a producer from a fork, that is already
-        # connected to Kafka. We allow forking producers only before they are used
-        raise Errors::ProducerUsedInParentProcess, Process.pid if @status.connected?
-
         # We undefine all the finalizers, in case it was a fork, so the finalizers from the parent
         # process don't leak
         ObjectSpace.undefine_finalizer(id)
+
+        # We should raise an error when trying to use a producer with client from a fork. Always.
+        if @client
+          # We need to reset the client, otherwise there might be attempt to close the parent
+          # client
+          @client = nil
+          raise Errors::ProducerUsedInParentProcess, Process.pid
+        end
+
         # Finalizer tracking is needed for handling shutdowns gracefully.
         # I don't expect everyone to remember about closing all the producers all the time, thus
         # this approach is better. Although it is still worth keeping in mind, that this will
@@ -137,7 +143,7 @@ module WaterDrop
           # It is safe to run it several times but not exactly the same moment
           # We also mark it as closed only if it was connected, if not, it would trigger a new
           # connection that anyhow would be immediately closed
-          client.close(@config.max_wait_timeout) if @client
+          client.close if @client
 
           # Remove callbacks runners that were registered
           ::Karafka::Core::Instrumentation.statistics_callbacks.delete(@id)
@@ -168,15 +174,35 @@ module WaterDrop
       @contract.validate!(message, Errors::MessageInvalidError)
     end
 
+    # Waits on a given handler
+    #
+    # @param handler [Rdkafka::Producer::DeliveryHandle]
+    def wait(handler)
+      handler.wait(
+        max_wait_timeout: @config.max_wait_timeout,
+        wait_timeout: @config.wait_timeout
+      )
+    end
+
+    private
+
     # Runs the client produce method with a given message
     #
     # @param message [Hash] message we want to send
     def produce(message)
+      produce_time ||= monotonic_now
+
       client.produce(**message)
     rescue SUPPORTED_FLOW_ERRORS.first => e
       # Unless we want to wait and retry and it's a full queue, we raise normally
       raise unless @config.wait_on_queue_full
       raise unless e.code == :queue_full
+      # If we're running for longer than the timeout, we need to re-raise the queue full.
+      # This will prevent from situation where cluster is down forever and we just retry and retry
+      # in an infinite loop, effectively hanging the processing
+      raise unless monotonic_now - produce_time < @config.wait_timeout_on_queue_full * 1_000
+
+      label = caller_locations(2, 1)[0].label.split(' ').last
 
       # We use this syntax here because we want to preserve the original `#cause` when we
       # instrument the error and there is no way to manually assign `#cause` value. We want to keep
@@ -195,25 +221,15 @@ module WaterDrop
           producer_id: id,
           message: message,
           error: e,
-          type: 'message.produce'
+          type: "message.#{label}"
         )
 
         # We do not poll the producer because polling happens in a background thread
         # It also should not be a frequent case (queue full), hence it's ok to just throttle.
-        sleep @config.wait_on_queue_full_timeout
+        sleep @config.wait_backoff_on_queue_full
       end
 
       retry
     end
-
-    # Waits on a given handler
-    #
-    # @param handler [Rdkafka::Producer::DeliveryHandle]
-    def wait(handler)
-      handler.wait(
-        max_wait_timeout: @config.max_wait_timeout,
-        wait_timeout: @config.wait_timeout
-      )
-    end
   end
 end

data/lib/waterdrop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '2.5.3'
+  VERSION = '2.6.1.beta1'
 end

data/waterdrop.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |spec|
   spec.description   = spec.summary
   spec.license       = 'MIT'
 
-  spec.add_dependency 'karafka-core', '>= 2.0.13', '< 3.0.0'
+  spec.add_dependency 'karafka-core', '>= 2.1.0.beta1', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   if $PROGRAM_NAME.end_with?('gem')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 2.5.3
+  version: 2.6.1.beta1
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2023-05-26 00:00:00.000000000 Z
+date: 2023-06-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.13
+        version: 2.1.0.beta1
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -53,7 +53,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.13
+        version: 2.1.0.beta1
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -95,6 +95,9 @@ files:
 - config/locales/errors.yml
 - docker-compose.yml
 - lib/waterdrop.rb
+- lib/waterdrop/clients/buffered.rb
+- lib/waterdrop/clients/dummy.rb
+- lib/waterdrop/clients/rdkafka.rb
 - lib/waterdrop/config.rb
 - lib/waterdrop/contracts.rb
 - lib/waterdrop/contracts/config.rb
@@ -109,14 +112,10 @@ files:
 - lib/waterdrop/instrumentation/vendors/datadog/dashboard.json
 - lib/waterdrop/instrumentation/vendors/datadog/metrics_listener.rb
 - lib/waterdrop/middleware.rb
-- lib/waterdrop/patches/rdkafka/client.rb
-- lib/waterdrop/patches/rdkafka/metadata.rb
-- lib/waterdrop/patches/rdkafka/producer.rb
 - lib/waterdrop/producer.rb
 - lib/waterdrop/producer/async.rb
 - lib/waterdrop/producer/buffer.rb
 - lib/waterdrop/producer/builder.rb
-- lib/waterdrop/producer/dummy_client.rb
 - lib/waterdrop/producer/status.rb
 - lib/waterdrop/producer/sync.rb
 - lib/waterdrop/version.rb
@@ -145,9 +144,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.4.10
 signing_key: 

metadata.gz.sig

@@ -1,3 +1 @@
-
-y��-�����ħԎ���+��V^�`ͰT���T��(!è��'D.7�jJ�����_�h�(P4��(6&��?OJȦD}���xd'��5�
-�����n�s�	�)�c�l:�v)���0����pS�"��
�R�W�̢}�����؝�g��Qb�9�T���5�S5�F��^�
+Z9}O��Tg+��zvQ�7�w����jh%/4�A�Ѱc�)�0A��)��u~�.]�!S�

data/lib/waterdrop/patches/rdkafka/client.rb

@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-module WaterDrop
-  module Patches
-    module Rdkafka
-      # Patches for the producer client
-      module Client
-        # @param _object_id [nil] rdkafka API compatibility argument
-        # @param timeout_ms [Integer] final flush timeout in ms
-        def close(_object_id = nil, timeout_ms = 5_000)
-          return unless @native
-
-          # Indicate to polling thread that we're closing
-          @polling_thread[:closing] = true
-          # Wait for the polling thread to finish up
-          @polling_thread.join
-
-          ::Rdkafka::Bindings.rd_kafka_flush(@native, timeout_ms)
-          ::Rdkafka::Bindings.rd_kafka_destroy(@native)
-
-          @native = nil
-        end
-      end
-    end
-  end
-end
-
-::Rdkafka::Bindings.attach_function(
-  :rd_kafka_flush,
-  %i[pointer int],
-  :void
-)
-
-Rdkafka::Producer::Client.prepend WaterDrop::Patches::Rdkafka::Client

data/lib/waterdrop/patches/rdkafka/metadata.rb

@@ -1,45 +0,0 @@
-# frozen_string_literal: true
-
-module WaterDrop
-  # Patches to external components
-  module Patches
-    # Rdkafka related patches
-    module Rdkafka
-      # Rdkafka::Metadata patches
-      module Metadata
-        # Errors upon which we retry the metadata fetch
-        RETRIED_ERRORS = %i[
-          timed_out
-          leader_not_available
-        ].freeze
-
-        private_constant :RETRIED_ERRORS
-
-        # We overwrite this method because there were reports of metadata operation timing out
-        # when Kafka was under stress. While the messages dispatch will be retried, metadata
-        # fetch happens prior to that, effectively crashing the process. Metadata fetch was not
-        # being retried at all.
-        #
-        # @param args [Array<Object>] all the metadata original arguments
-        def initialize(*args)
-          attempt ||= 0
-          attempt += 1
-
-          super(*args)
-        rescue ::Rdkafka::RdkafkaError => e
-          raise unless RETRIED_ERRORS.include?(e.code)
-          raise if attempt > 10
-
-          backoff_factor = 2**attempt
-          timeout = backoff_factor * 0.1
-
-          sleep(timeout)
-
-          retry
-        end
-      end
-    end
-  end
-end
-
-::Rdkafka::Metadata.prepend ::WaterDrop::Patches::Rdkafka::Metadata

data/lib/waterdrop/patches/rdkafka/producer.rb

@@ -1,86 +0,0 @@
-# frozen_string_literal: true
-
-module WaterDrop
-  # Patches to external components
-  module Patches
-    # Rdkafka related patches
-    module Rdkafka
-      # Rdkafka::Producer patches
-      module Producer
-        include ::Karafka::Core::Helpers::Time
-
-        # Cache partitions count for 30 seconds
-        PARTITIONS_COUNT_TTL = 30_000
-
-        private_constant :PARTITIONS_COUNT_TTL
-
-        # @param args [Object] arguments accepted by the original rdkafka producer
-        def initialize(*args)
-          super
-
-          @_partitions_count_cache = Concurrent::Hash.new do |cache, topic|
-            topic_metadata = ::Rdkafka::Metadata.new(inner_kafka, topic).topics&.first
-
-            cache[topic] = [
-              monotonic_now,
-              topic_metadata ? topic_metadata[:partition_count] : nil
-            ]
-          end
-        end
-
-        # Adds a method that allows us to get the native kafka producer name
-        #
-        # In between rdkafka versions, there are internal changes that force us to add some extra
-        # magic to support all the versions.
-        #
-        # @return [String] producer instance name
-        def name
-          @_name ||= ::Rdkafka::Bindings.rd_kafka_name(inner_kafka)
-        end
-
-        # This patch makes sure we cache the partition count for a given topic for given time
-        # This prevents us in case someone uses `partition_key` from querying for the count with
-        # each message. Instead we query once every 30 seconds at most
-        #
-        # @param topic [String] topic name
-        # @return [Integer] partition count for a given topic
-        def partition_count(topic)
-          closed_producer_check(__method__)
-
-          @_partitions_count_cache.delete_if do |_, cached|
-            monotonic_now - cached.first > PARTITIONS_COUNT_TTL
-          end
-
-          @_partitions_count_cache[topic].last
-        end
-
-        # @return [FFI::Pointer] pointer to the raw librdkafka
-        def inner_kafka
-          unless @_inner_kafka
-            version = ::Gem::Version.new(::Rdkafka::VERSION)
-
-            if version < ::Gem::Version.new('0.12.0')
-              @_inner_kafka = @native_kafka
-            elsif version < ::Gem::Version.new('0.13.0.beta.1')
-              @_inner_kafka = @client.native
-            else
-              @_inner_kafka = @native_kafka.inner
-            end
-          end
-
-          @_inner_kafka
-        end
-
-        # Closes our librdkafka instance with the flush patch
-        # @param timeout_ms [Integer] flush timeout
-        def close(timeout_ms = 5_000)
-          ObjectSpace.undefine_finalizer(self)
-
-          @client.close(nil, timeout_ms)
-        end
-      end
-    end
-  end
-end
-
-::Rdkafka::Producer.prepend ::WaterDrop::Patches::Rdkafka::Producer