waterdrop 2.6.1.beta1 → 2.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5e637029ce9d9113e2d64cadcec57ac980602d489f2207507194082065604b16
-  data.tar.gz: 21d0a5a643b3e48603fe8e8adb13dbe827a0b9b50871675e57aa62b71abe7573
+  metadata.gz: 1ed8c6f17cd730ab82df21e696e472c51dd51379acf757f735d43de98c00dead
+  data.tar.gz: 10f8b0d7bd64c26ba9b76c9edb76cf59f871cf44adbde2198907856f7b761ea4
 SHA512:
-  metadata.gz: e34c6c5225022f191dd80da877126b5c37075b3356b1e99e4e6b79adb0c54b8939e6b43f2efded8f68a0b68424b3cdbdcaa8abe19a781456faa4f653ae9063ea
-  data.tar.gz: 5060d94c549d5d0ac4b2e09f8ab4888643008ea732943e76f75dfedd386f08ada8a60ad278f85dcfa1172bf813c993a38b2feb71ee75ea2ce1a72bf3021edbcb
+  metadata.gz: a41f9a5e1ef30c69221615552f48bad03099697d27a1e405f90ab189f6c21a33dedc6382c864d4790a22d798c64023a1802597f8ce255214d077de0ebcd5028b
+  data.tar.gz: b21a71b53e188fd58ce8800bc23548cc3bbc0a876fe4aa8d9c67107b0e90ddfd6484c4e534137b63e4bb45d5760460ae4b7df79233d6a2acc85ba00d45a55e2a

data/CHANGELOG.md

@@ -1,9 +1,16 @@
 # WaterDrop changelog
 
-### 2.6.1 (Unreleased)
+### 2.6.2 (2023-06-21)
+- [Refactor] Introduce a counter-based locking approach to make sure, that we close the producer safely but at the same time not to limit messages production with producing lock.
+- [Refactor] Make private methods private.
+- [Refactor] Validate that producer is not closed only when attempting to produce.
+- [Refactor] Improve one 5 minute long spec to run in 10 seconds.
+- [Refactor] clear client assignment after closing.
+
+### 2.6.1 (2023-06-19)
 - [Refactor] Remove no longer needed patches.
 - [Fix] Fork detection on a short lived processes seems to fail. Clear the used parent process client reference not to close it in the finalizer (#356).
-- [Change] Require `karafka-rdkafka` `>= 0.13.0.beta2`.
+- [Change] Require `karafka-rdkafka` `>= 0.13.0`.
 - [Change] Require 'karafka-core' `>= 2.1.0`
 
 ### 2.6.0 (2023-06-11)

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.6.1.beta1)
-      karafka-core (>= 2.1.0.beta1, < 3.0.0)
+    waterdrop (2.6.2)
+      karafka-core (>= 2.1.0, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
@@ -22,10 +22,10 @@ GEM
     ffi (1.15.5)
     i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.1.0.beta1)
+    karafka-core (2.1.0)
       concurrent-ruby (>= 1.1)
-      karafka-rdkafka (>= 0.13.0.beta2, < 0.14.0)
-    karafka-rdkafka (0.13.0.beta2)
+      karafka-rdkafka (>= 0.13.0, < 0.14.0)
+    karafka-rdkafka (0.13.0)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)

data/lib/waterdrop/clients/buffered.rb

@@ -24,8 +24,8 @@ module WaterDrop
       # "Produces" message to Kafka: it acknowledges it locally, adds it to the internal buffer
       # @param message [Hash] `WaterDrop::Producer#produce_sync` message hash
       def produce(message)
-        topic = message.fetch(:topic) { raise ArgumentError, ':topic is missing' }
-        @topics[topic] << message
+        # We pre-validate the message payload, so topic is ensured to be present
+        @topics[message.fetch(:topic)] << message
         @messages << message
         SyncResponse.new
       end

data/lib/waterdrop/producer/async.rb

@@ -14,8 +14,6 @@ module WaterDrop
       # @raise [Errors::MessageInvalidError] When provided message details are invalid and the
       #   message could not be sent to Kafka
       def produce_async(message)
-        ensure_active!
-
         message = middleware.run(message)
         validate_message!(message)
 
@@ -49,8 +47,6 @@ module WaterDrop
       # @raise [Errors::MessageInvalidError] When any of the provided messages details are invalid
       #   and the message could not be sent to Kafka
       def produce_many_async(messages)
-        ensure_active!
-
         dispatched = []
         messages = middleware.run_many(messages)
         messages.each { |message| validate_message!(message) }

data/lib/waterdrop/producer/buffer.rb

@@ -11,6 +11,7 @@ module WaterDrop
       #   message could not be sent to Kafka
       def buffer(message)
         ensure_active!
+
         message = middleware.run(message)
         validate_message!(message)
 
@@ -49,8 +50,6 @@ module WaterDrop
       # @return [Array<Rdkafka::Producer::DeliveryHandle>] delivery handles for messages that were
       #   flushed
       def flush_async
-        ensure_active!
-
         @monitor.instrument(
           'buffer.flushed_async',
           producer_id: id,
@@ -62,8 +61,6 @@ module WaterDrop
       # @return [Array<Rdkafka::Producer::DeliveryReport>] delivery reports for messages that were
       #   flushed
       def flush_sync
-        ensure_active!
-
         @monitor.instrument(
           'buffer.flushed_sync',
           producer_id: id,

data/lib/waterdrop/producer/sync.rb

@@ -16,8 +16,6 @@ module WaterDrop
       # @raise [Errors::MessageInvalidError] When provided message details are invalid and the
       #   message could not be sent to Kafka
       def produce_sync(message)
-        ensure_active!
-
         message = middleware.run(message)
         validate_message!(message)
 
@@ -55,8 +53,6 @@ module WaterDrop
       # @raise [Errors::MessageInvalidError] When any of the provided messages details are invalid
       #   and the message could not be sent to Kafka
       def produce_many_sync(messages)
-        ensure_active! unless @closing_thread_id && @closing_thread_id == Thread.current.object_id
-
         messages = middleware.run_many(messages)
         messages.each { |message| validate_message!(message) }
 

data/lib/waterdrop/producer.rb

@@ -34,9 +34,10 @@ module WaterDrop
     # @param block [Proc] configuration block
     # @return [Producer] producer instance
     def initialize(&block)
+      @operations_in_progress = 0
       @buffer_mutex = Mutex.new
       @connecting_mutex = Mutex.new
-      @closing_mutex = Mutex.new
+      @operating_mutex = Mutex.new
 
       @status = Status.new
       @messages = Concurrent::Array.new
@@ -118,7 +119,7 @@ module WaterDrop
 
     # Flushes the buffers in a sync way and closes the producer
     def close
-      @closing_mutex.synchronize do
+      @operating_mutex.synchronize do
         return unless @status.active?
 
         @monitor.instrument(
@@ -135,6 +136,10 @@ module WaterDrop
           # producer for final flush of buffers.
           @closing_thread_id = Thread.current.object_id
 
+          # Wait until all the outgoing operations are done. Only when no one is using the
+          # underlying client running operations we can close
+          sleep(0.001) until @operations_in_progress.zero?
+
           # Flush has its own buffer mutex but even if it is blocked, flushing can still happen
           # as we close the client after the flushing (even if blocked by the mutex)
           flush(true)
@@ -143,7 +148,10 @@ module WaterDrop
           # It is safe to run it several times but not exactly the same moment
           # We also mark it as closed only if it was connected, if not, it would trigger a new
           # connection that anyhow would be immediately closed
-          client.close if @client
+          if @client
+            client.close
+            @client = nil
+          end
 
           # Remove callbacks runners that were registered
           ::Karafka::Core::Instrumentation.statistics_callbacks.delete(@id)
@@ -154,13 +162,17 @@ module WaterDrop
       end
     end
 
+    private
+
     # Ensures that we don't run any operations when the producer is not configured or when it
     # was already closed
     def ensure_active!
       return if @status.active?
+      return if @status.closing? && @operating_mutex.owned?
 
       raise Errors::ProducerNotConfiguredError, id if @status.initial?
-      raise Errors::ProducerClosedError, id if @status.closing? || @status.closed?
+      raise Errors::ProducerClosedError, id if @status.closing?
+      raise Errors::ProducerClosedError, id if @status.closed?
 
       # This should never happen
       raise Errors::StatusInvalidError, [id, @status.to_s]
@@ -184,14 +196,21 @@ module WaterDrop
       )
     end
 
-    private
-
     # Runs the client produce method with a given message
     #
     # @param message [Hash] message we want to send
     def produce(message)
       produce_time ||= monotonic_now
 
+      # This can happen only during flushing on closing, in case like this we don't have to
+      # synchronize because we already own the lock
+      if @operating_mutex.owned?
+        @operations_in_progress += 1
+      else
+        @operating_mutex.synchronize { @operations_in_progress += 1 }
+        ensure_active!
+      end
+
       client.produce(**message)
     rescue SUPPORTED_FLOW_ERRORS.first => e
       # Unless we want to wait and retry and it's a full queue, we raise normally
@@ -229,7 +248,10 @@ module WaterDrop
         sleep @config.wait_backoff_on_queue_full
       end
 
+      @operations_in_progress -= 1
       retry
+    ensure
+      @operations_in_progress -= 1
     end
   end
 end

data/lib/waterdrop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '2.6.1.beta1'
+  VERSION = '2.6.2'
 end

data/waterdrop.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |spec|
   spec.description   = spec.summary
   spec.license       = 'MIT'
 
-  spec.add_dependency 'karafka-core', '>= 2.1.0.beta1', '< 3.0.0'
+  spec.add_dependency 'karafka-core', '>= 2.1.0', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   if $PROGRAM_NAME.end_with?('gem')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 2.6.1.beta1
+  version: 2.6.2
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2023-06-17 00:00:00.000000000 Z
+date: 2023-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.1.0.beta1
+        version: 2.1.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -53,7 +53,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.1.0.beta1
+        version: 2.1.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -144,9 +144,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.4.10
 signing_key: 

metadata.gz.sig

@@ -1 +1 @@
-Z9}O��Tg+��zvQ�7�w����jh%/4�A�Ѱc�)�0A��)��u~�.]�!S�
+E���y���!+t �V�	����5cK|��3��=�$š�x�3KN$ R<�T���=�{60�-�V����=���6���w�}�Ff%r�Z��������s�P��X��)�]��