waterdrop 2.6.1 → 2.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2150248405c747e29acb8e31207be9dc0c7c262916f003af9c55208ee3675d3d
-  data.tar.gz: 0c920351ac4a40db3a0c57e8f4c12fa6f3c75ee197b9696c5b6464a35c1a8889
+  metadata.gz: f051a62baab08b5b23bb85a5b2264f3b33bebb77c0f3628ed90eda5eda8c22f2
+  data.tar.gz: 9a5f27b24bee02bb2efcaeec926a8568c540c420c77cf1c9a06a6d72fcf496b1
 SHA512:
-  metadata.gz: e0eab9080a356e2d3ee5c2c68d48484057d4ed77aebcc9eda0a88c28f5e4da7812072686b60c12eb6f2536a1c4daa8b427844f2abbeccbfaf22f18f597702707
-  data.tar.gz: 462fbb70e2794703b5d09b29b94c1c1c6b8e8642beb407f1dee638be66913a07af8c92114d129d1c85ad8c8f03e7d1ea776be42b0d0daa37b801aba607d40cf4
+  metadata.gz: 7fd5cd82e391ad95e9c48b4e6794ad6b4e0715b6f4a64e838c676064b96e22c2c0a0cea1f18e4131c5dc60ae33923b943d2828278404038f5f7a79b754cb6e37
+  data.tar.gz: 004d5f3fdbbe48733ba47f132e12dcf1bd3ab5e0705181825f8e93b207e12b955b97cf6c283dcb58049153dd3325dd3ea655130e60982d0300ec309460e0d583

checksums.yaml.gz.sig

@@ -1,2 +1,4 @@
-����[�N,Lbn���$�E�V��)�7g��$��6h�2@MӖn���X�H!��A����B�G�z&
gkP߶ꮐ4����)��k��e%���eJ]#�D�3�*p���<�m_����k�I���yz-��v�ӷ�
-��B*k6՝��Y!�^� ��AUr�C44�ZD���
�G�����'�	��Z�]�#DMͽ��S�=�]�
����ykr�M4����=��"��X�0k��<�D�>��%��{�c1;(�Rʩ}�Gf�m�'�a�� 9
+�N>�c��
��(�P}cRMo4�xB�=_qQ`��YG��+�v
X�
+��+rH�AT�Wq��\�����]h3�X��f��=�'{��Cᮄ��hRp8Ҵ e8�H=D��3�{y;����/�Bqkm�+�q�?'6�*q�S| ,ԡl�tg���	�+�����"��8�\�3�7#߀)PZ��(����7�����*Y&3
+!J�5F�%��VҾ5�s^q�6dy���(�4�^�m+�?�x�g )�ގt���9�u��7Q�7o�$�����k�
+�����a���+vT�g7��]o,f�&�X)�?

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 # WaterDrop changelog
 
+### 2.6.3 (2023-06-28)
+- [Change] Use `Concurrent::AtomicFixnum` to track operations in progress to prevent potential race conditions on JRuby and TruffleRuby (not yet supported but this is for future usage).
+- [Change] Require `karafka-rdkafka` `>= 0.13.2`.
+- [Change] Require 'karafka-core' `>= 2.1.1`
+
+### 2.6.2 (2023-06-21)
+- [Refactor] Introduce a counter-based locking approach to make sure, that we close the producer safely but at the same time not to limit messages production with producing lock.
+- [Refactor] Make private methods private.
+- [Refactor] Validate that producer is not closed only when attempting to produce.
+- [Refactor] Improve one 5 minute long spec to run in 10 seconds.
+- [Refactor] clear client assignment after closing.
+
 ### 2.6.1 (2023-06-19)
 - [Refactor] Remove no longer needed patches.
 - [Fix] Fork detection on a short lived processes seems to fail. Clear the used parent process client reference not to close it in the finalizer (#356).

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.6.1)
-      karafka-core (>= 2.1.0, < 3.0.0)
+    waterdrop (2.6.3)
+      karafka-core (>= 2.1.1, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
@@ -22,10 +22,10 @@ GEM
     ffi (1.15.5)
     i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.1.0)
+    karafka-core (2.1.1)
       concurrent-ruby (>= 1.1)
-      karafka-rdkafka (>= 0.13.0, < 0.14.0)
-    karafka-rdkafka (0.13.0)
+      karafka-rdkafka (>= 0.13.1, < 0.14.0)
+    karafka-rdkafka (0.13.2)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)

data/lib/waterdrop/clients/buffered.rb

@@ -24,8 +24,8 @@ module WaterDrop
       # "Produces" message to Kafka: it acknowledges it locally, adds it to the internal buffer
       # @param message [Hash] `WaterDrop::Producer#produce_sync` message hash
       def produce(message)
-        topic = message.fetch(:topic) { raise ArgumentError, ':topic is missing' }
-        @topics[topic] << message
+        # We pre-validate the message payload, so topic is ensured to be present
+        @topics[message.fetch(:topic)] << message
         @messages << message
         SyncResponse.new
       end

data/lib/waterdrop/producer/async.rb

@@ -14,8 +14,6 @@ module WaterDrop
       # @raise [Errors::MessageInvalidError] When provided message details are invalid and the
       #   message could not be sent to Kafka
       def produce_async(message)
-        ensure_active!
-
         message = middleware.run(message)
         validate_message!(message)
 
@@ -49,8 +47,6 @@ module WaterDrop
       # @raise [Errors::MessageInvalidError] When any of the provided messages details are invalid
       #   and the message could not be sent to Kafka
       def produce_many_async(messages)
-        ensure_active!
-
         dispatched = []
         messages = middleware.run_many(messages)
         messages.each { |message| validate_message!(message) }

data/lib/waterdrop/producer/buffer.rb

@@ -11,6 +11,7 @@ module WaterDrop
       #   message could not be sent to Kafka
       def buffer(message)
         ensure_active!
+
         message = middleware.run(message)
         validate_message!(message)
 
@@ -49,8 +50,6 @@ module WaterDrop
       # @return [Array<Rdkafka::Producer::DeliveryHandle>] delivery handles for messages that were
       #   flushed
       def flush_async
-        ensure_active!
-
         @monitor.instrument(
           'buffer.flushed_async',
           producer_id: id,
@@ -62,8 +61,6 @@ module WaterDrop
       # @return [Array<Rdkafka::Producer::DeliveryReport>] delivery reports for messages that were
       #   flushed
       def flush_sync
-        ensure_active!
-
         @monitor.instrument(
           'buffer.flushed_sync',
           producer_id: id,

data/lib/waterdrop/producer/sync.rb

@@ -16,8 +16,6 @@ module WaterDrop
       # @raise [Errors::MessageInvalidError] When provided message details are invalid and the
       #   message could not be sent to Kafka
       def produce_sync(message)
-        ensure_active!
-
         message = middleware.run(message)
         validate_message!(message)
 
@@ -55,8 +53,6 @@ module WaterDrop
       # @raise [Errors::MessageInvalidError] When any of the provided messages details are invalid
       #   and the message could not be sent to Kafka
       def produce_many_sync(messages)
-        ensure_active! unless @closing_thread_id && @closing_thread_id == Thread.current.object_id
-
         messages = middleware.run_many(messages)
         messages.each { |message| validate_message!(message) }
 

data/lib/waterdrop/producer.rb

@@ -34,9 +34,10 @@ module WaterDrop
     # @param block [Proc] configuration block
     # @return [Producer] producer instance
     def initialize(&block)
+      @operations_in_progress = Concurrent::AtomicFixnum.new(0)
       @buffer_mutex = Mutex.new
       @connecting_mutex = Mutex.new
-      @closing_mutex = Mutex.new
+      @operating_mutex = Mutex.new
 
       @status = Status.new
       @messages = Concurrent::Array.new
@@ -118,7 +119,7 @@ module WaterDrop
 
     # Flushes the buffers in a sync way and closes the producer
     def close
-      @closing_mutex.synchronize do
+      @operating_mutex.synchronize do
         return unless @status.active?
 
         @monitor.instrument(
@@ -135,6 +136,10 @@ module WaterDrop
           # producer for final flush of buffers.
           @closing_thread_id = Thread.current.object_id
 
+          # Wait until all the outgoing operations are done. Only when no one is using the
+          # underlying client running operations we can close
+          sleep(0.001) until @operations_in_progress.value.zero?
+
           # Flush has its own buffer mutex but even if it is blocked, flushing can still happen
           # as we close the client after the flushing (even if blocked by the mutex)
           flush(true)
@@ -143,7 +148,10 @@ module WaterDrop
           # It is safe to run it several times but not exactly the same moment
           # We also mark it as closed only if it was connected, if not, it would trigger a new
           # connection that anyhow would be immediately closed
-          client.close if @client
+          if @client
+            client.close
+            @client = nil
+          end
 
           # Remove callbacks runners that were registered
           ::Karafka::Core::Instrumentation.statistics_callbacks.delete(@id)
@@ -154,13 +162,17 @@ module WaterDrop
       end
     end
 
+    private
+
     # Ensures that we don't run any operations when the producer is not configured or when it
     # was already closed
     def ensure_active!
       return if @status.active?
+      return if @status.closing? && @operating_mutex.owned?
 
       raise Errors::ProducerNotConfiguredError, id if @status.initial?
-      raise Errors::ProducerClosedError, id if @status.closing? || @status.closed?
+      raise Errors::ProducerClosedError, id if @status.closing?
+      raise Errors::ProducerClosedError, id if @status.closed?
 
       # This should never happen
       raise Errors::StatusInvalidError, [id, @status.to_s]
@@ -184,14 +196,21 @@ module WaterDrop
       )
     end
 
-    private
-
     # Runs the client produce method with a given message
     #
     # @param message [Hash] message we want to send
     def produce(message)
       produce_time ||= monotonic_now
 
+      # This can happen only during flushing on closing, in case like this we don't have to
+      # synchronize because we already own the lock
+      if @operating_mutex.owned?
+        @operations_in_progress.increment
+      else
+        @operating_mutex.synchronize { @operations_in_progress.increment }
+        ensure_active!
+      end
+
       client.produce(**message)
     rescue SUPPORTED_FLOW_ERRORS.first => e
       # Unless we want to wait and retry and it's a full queue, we raise normally
@@ -229,7 +248,10 @@ module WaterDrop
         sleep @config.wait_backoff_on_queue_full
       end
 
+      @operations_in_progress.decrement
       retry
+    ensure
+      @operations_in_progress.decrement
     end
   end
 end

data/lib/waterdrop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '2.6.1'
+  VERSION = '2.6.3'
 end

data/lib/waterdrop.rb

@@ -9,6 +9,7 @@
   securerandom
   karafka-core
   pathname
+  concurrent/atomic/atomic_fixnum
 ].each { |lib| require lib }
 
 # WaterDrop library

data/waterdrop.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |spec|
   spec.description   = spec.summary
   spec.license       = 'MIT'
 
-  spec.add_dependency 'karafka-core', '>= 2.1.0', '< 3.0.0'
+  spec.add_dependency 'karafka-core', '>= 2.1.1', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   if $PROGRAM_NAME.end_with?('gem')

data.tar.gz.sig

@@ -1,3 +1,4 @@
-=f��~�N��?bƉ�oߨ�bqc�}�ݵ=�#�L��nA�'�Q�o<Mw,���xY��rb�z�]�6�M������RV���\�[�|]�]��f�H�z8�O}���mK�
-����Pע��Oeǩ��
-<+����݊'zD�dv��o�;���<&8J,�8����A/��͔�=3�rH��3
��!+��U#��J�����
+#�}��6���ӝߕ�4
+�
+�01\�������	.)dF��J�bvN,
+��Y,�e�i�i�g�Q&��

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 2.6.1
+  version: 2.6.3
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2023-06-19 00:00:00.000000000 Z
+date: 2023-06-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: 2.1.1
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -53,7 +53,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: 2.1.1
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0