waterdrop 2.6.1 → 2.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2150248405c747e29acb8e31207be9dc0c7c262916f003af9c55208ee3675d3d
-  data.tar.gz: 0c920351ac4a40db3a0c57e8f4c12fa6f3c75ee197b9696c5b6464a35c1a8889
+  metadata.gz: 1ed8c6f17cd730ab82df21e696e472c51dd51379acf757f735d43de98c00dead
+  data.tar.gz: 10f8b0d7bd64c26ba9b76c9edb76cf59f871cf44adbde2198907856f7b761ea4
 SHA512:
-  metadata.gz: e0eab9080a356e2d3ee5c2c68d48484057d4ed77aebcc9eda0a88c28f5e4da7812072686b60c12eb6f2536a1c4daa8b427844f2abbeccbfaf22f18f597702707
-  data.tar.gz: 462fbb70e2794703b5d09b29b94c1c1c6b8e8642beb407f1dee638be66913a07af8c92114d129d1c85ad8c8f03e7d1ea776be42b0d0daa37b801aba607d40cf4
+  metadata.gz: a41f9a5e1ef30c69221615552f48bad03099697d27a1e405f90ab189f6c21a33dedc6382c864d4790a22d798c64023a1802597f8ce255214d077de0ebcd5028b
+  data.tar.gz: b21a71b53e188fd58ce8800bc23548cc3bbc0a876fe4aa8d9c67107b0e90ddfd6484c4e534137b63e4bb45d5760460ae4b7df79233d6a2acc85ba00d45a55e2a

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # WaterDrop changelog
 
+### 2.6.2 (2023-06-21)
+- [Refactor] Introduce a counter-based locking approach to make sure, that we close the producer safely but at the same time not to limit messages production with producing lock.
+- [Refactor] Make private methods private.
+- [Refactor] Validate that producer is not closed only when attempting to produce.
+- [Refactor] Improve one 5 minute long spec to run in 10 seconds.
+- [Refactor] clear client assignment after closing.
+
 ### 2.6.1 (2023-06-19)
 - [Refactor] Remove no longer needed patches.
 - [Fix] Fork detection on a short lived processes seems to fail. Clear the used parent process client reference not to close it in the finalizer (#356).

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.6.1)
+    waterdrop (2.6.2)
       karafka-core (>= 2.1.0, < 3.0.0)
       zeitwerk (~> 2.3)
 

data/lib/waterdrop/clients/buffered.rb

@@ -24,8 +24,8 @@ module WaterDrop
       # "Produces" message to Kafka: it acknowledges it locally, adds it to the internal buffer
       # @param message [Hash] `WaterDrop::Producer#produce_sync` message hash
       def produce(message)
-        topic = message.fetch(:topic) { raise ArgumentError, ':topic is missing' }
-        @topics[topic] << message
+        # We pre-validate the message payload, so topic is ensured to be present
+        @topics[message.fetch(:topic)] << message
         @messages << message
         SyncResponse.new
       end

data/lib/waterdrop/producer/async.rb

@@ -14,8 +14,6 @@ module WaterDrop
       # @raise [Errors::MessageInvalidError] When provided message details are invalid and the
       #   message could not be sent to Kafka
       def produce_async(message)
-        ensure_active!
-
         message = middleware.run(message)
         validate_message!(message)
 
@@ -49,8 +47,6 @@ module WaterDrop
       # @raise [Errors::MessageInvalidError] When any of the provided messages details are invalid
       #   and the message could not be sent to Kafka
       def produce_many_async(messages)
-        ensure_active!
-
         dispatched = []
         messages = middleware.run_many(messages)
         messages.each { |message| validate_message!(message) }

data/lib/waterdrop/producer/buffer.rb

@@ -11,6 +11,7 @@ module WaterDrop
       #   message could not be sent to Kafka
       def buffer(message)
         ensure_active!
+
         message = middleware.run(message)
         validate_message!(message)
 
@@ -49,8 +50,6 @@ module WaterDrop
       # @return [Array<Rdkafka::Producer::DeliveryHandle>] delivery handles for messages that were
       #   flushed
       def flush_async
-        ensure_active!
-
         @monitor.instrument(
           'buffer.flushed_async',
           producer_id: id,
@@ -62,8 +61,6 @@ module WaterDrop
       # @return [Array<Rdkafka::Producer::DeliveryReport>] delivery reports for messages that were
       #   flushed
       def flush_sync
-        ensure_active!
-
         @monitor.instrument(
           'buffer.flushed_sync',
           producer_id: id,

data/lib/waterdrop/producer/sync.rb

@@ -16,8 +16,6 @@ module WaterDrop
       # @raise [Errors::MessageInvalidError] When provided message details are invalid and the
       #   message could not be sent to Kafka
       def produce_sync(message)
-        ensure_active!
-
         message = middleware.run(message)
         validate_message!(message)
 
@@ -55,8 +53,6 @@ module WaterDrop
       # @raise [Errors::MessageInvalidError] When any of the provided messages details are invalid
       #   and the message could not be sent to Kafka
       def produce_many_sync(messages)
-        ensure_active! unless @closing_thread_id && @closing_thread_id == Thread.current.object_id
-
         messages = middleware.run_many(messages)
         messages.each { |message| validate_message!(message) }
 

data/lib/waterdrop/producer.rb

@@ -34,9 +34,10 @@ module WaterDrop
     # @param block [Proc] configuration block
     # @return [Producer] producer instance
     def initialize(&block)
+      @operations_in_progress = 0
       @buffer_mutex = Mutex.new
       @connecting_mutex = Mutex.new
-      @closing_mutex = Mutex.new
+      @operating_mutex = Mutex.new
 
       @status = Status.new
       @messages = Concurrent::Array.new
@@ -118,7 +119,7 @@ module WaterDrop
 
     # Flushes the buffers in a sync way and closes the producer
     def close
-      @closing_mutex.synchronize do
+      @operating_mutex.synchronize do
         return unless @status.active?
 
         @monitor.instrument(
@@ -135,6 +136,10 @@ module WaterDrop
           # producer for final flush of buffers.
           @closing_thread_id = Thread.current.object_id
 
+          # Wait until all the outgoing operations are done. Only when no one is using the
+          # underlying client running operations we can close
+          sleep(0.001) until @operations_in_progress.zero?
+
           # Flush has its own buffer mutex but even if it is blocked, flushing can still happen
           # as we close the client after the flushing (even if blocked by the mutex)
           flush(true)
@@ -143,7 +148,10 @@ module WaterDrop
           # It is safe to run it several times but not exactly the same moment
           # We also mark it as closed only if it was connected, if not, it would trigger a new
           # connection that anyhow would be immediately closed
-          client.close if @client
+          if @client
+            client.close
+            @client = nil
+          end
 
           # Remove callbacks runners that were registered
           ::Karafka::Core::Instrumentation.statistics_callbacks.delete(@id)
@@ -154,13 +162,17 @@ module WaterDrop
       end
     end
 
+    private
+
     # Ensures that we don't run any operations when the producer is not configured or when it
     # was already closed
     def ensure_active!
       return if @status.active?
+      return if @status.closing? && @operating_mutex.owned?
 
       raise Errors::ProducerNotConfiguredError, id if @status.initial?
-      raise Errors::ProducerClosedError, id if @status.closing? || @status.closed?
+      raise Errors::ProducerClosedError, id if @status.closing?
+      raise Errors::ProducerClosedError, id if @status.closed?
 
       # This should never happen
       raise Errors::StatusInvalidError, [id, @status.to_s]
@@ -184,14 +196,21 @@ module WaterDrop
       )
     end
 
-    private
-
     # Runs the client produce method with a given message
     #
     # @param message [Hash] message we want to send
     def produce(message)
       produce_time ||= monotonic_now
 
+      # This can happen only during flushing on closing, in case like this we don't have to
+      # synchronize because we already own the lock
+      if @operating_mutex.owned?
+        @operations_in_progress += 1
+      else
+        @operating_mutex.synchronize { @operations_in_progress += 1 }
+        ensure_active!
+      end
+
       client.produce(**message)
     rescue SUPPORTED_FLOW_ERRORS.first => e
       # Unless we want to wait and retry and it's a full queue, we raise normally
@@ -229,7 +248,10 @@ module WaterDrop
         sleep @config.wait_backoff_on_queue_full
       end
 
+      @operations_in_progress -= 1
       retry
+    ensure
+      @operations_in_progress -= 1
     end
   end
 end

data/lib/waterdrop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '2.6.1'
+  VERSION = '2.6.2'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 2.6.1
+  version: 2.6.2
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2023-06-19 00:00:00.000000000 Z
+date: 2023-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core

metadata.gz.sig

@@ -1,4 +1 @@
-T��:�h�ĎY��D
-C��4U9L��"tw
-�h����Al�� �T�:�-�m�SI��O��O���7V��I�gu�y$��2B�
-E=]�'},�G�|4a3���Q�Vk�i�4���=�k��qb�h�e�P��b��l��(�8���0�I~��I
4�Ŭ�I_ʏ^U8�{0���A{99���v��oeP�ܪ*z�� ��:k8VrB�fɹ����.��n����ࢀo���+�G6A���_}��l3�AJ���+Ul�SD��H�ۯ	�<>�I�%�WІ^wɔ*�M�s�f`�wk=���'hF�m��+��huYyu0
�}�2y�rq
+E���y���!+t �V�	����5cK|��3��=�$š�x�3KN$ R<�T���=�{60�-�V����=���6���w�}�Ff%r�Z��������s�P��X��)�]��