rdkafka 0.13.0.beta.7 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5a4ea932db3e81e325c31d9e1aedd5604eccc30d1faa865cac5986a66c59ddaa
-  data.tar.gz: b8da648c6ac7f83ba48c7e19a391e918dc4ed7d9bdc6ab57d9479b43b34760e3
+  metadata.gz: b4b1881445c53cdd213efd2e7ed3feb94a32a327fdf63ab35455d337f09e7320
+  data.tar.gz: 02276df9702539c951ab392b4c5e086f87b731f7d319aedafb7b1209bc444688
 SHA512:
-  metadata.gz: f4f25d641d18a7a1f2b2b6bc3f56ae9a56c26212a46d809f3d97a02bc41bf4bc1094d208762d3c3e96087222fa4ed6dfeacf4fab3967c93fd02540f999678f1d
-  data.tar.gz: 19f7832ccfce84daf5faca7eb3c74c37ab5dcdf7e60a40fed9cb10e99408a3baf61a310cdae9e63835b201b757428beb7b5c8f57cc1c82f6b99dd8b1bf280286
+  metadata.gz: 7f0e11a17d03091c8e13308c6998131c142349ce520473de7fec5908f0bcf2dfd59542966c3a850008f0db39e96b763b66f7b937487ba11f2fbb444e00aff25c
+  data.tar.gz: d15c5f7cd5f4b9041da55e7e46bf05936c627517d5067a9449844e2721e752de852006f432e5fbe72549d77d8378545a671a969652b4abada8aa8182f4fa4985

data/lib/rdkafka/consumer.rb

@@ -27,9 +27,11 @@ module Rdkafka
     def close
       return if closed?
       ObjectSpace.undefine_finalizer(self)
-      @native_kafka.with_inner do |inner|
+
+      @native_kafka.synchronize do |inner|
         Rdkafka::Bindings.rd_kafka_consumer_close(inner)
       end
+
       @native_kafka.close
     end
 

data/lib/rdkafka/error.rb

@@ -92,4 +92,10 @@ module Rdkafka
       super("Illegal call to #{method.to_s} on a closed admin")
     end
   end
+
+  class ClosedInnerError < BaseError
+    def initialize
+      super("Illegal call to a closed inner librdkafka instance")
+    end
+  end
 end

data/lib/rdkafka/native_kafka.rb

@@ -10,6 +10,22 @@ module Rdkafka
       @access_mutex = Mutex.new
       # Lock around internal polling
       @poll_mutex = Mutex.new
+      # Lock around decrementing the operations in progress counter
+      # We have two mutexes - one for increment (`@access_mutex`) and one for decrement mutex
+      # because they serve different purposes:
+      #
+      #   - `@access_mutex` allows us to lock the execution and make sure that any operation within
+      #      the `#synchronize` is the only one running and that there are no other running
+      #      operations.
+      #   - `@decrement_mutex` ensures, that our decrement operation is thread-safe for any Ruby
+      #     implementation.
+      #
+      # We do not use the same mutex, because it could create a deadlock when an already
+      # incremented operation cannot decrement because `@access_lock` is now owned by a different
+      # thread in a synchronized mode and the synchronized mode is waiting on the decrement.
+      @decrement_mutex = Mutex.new
+      # counter for operations in progress using inner
+      @operations_in_progress = 0
 
       if run_polling_thread
         # Start thread to poll client for delivery callbacks,
@@ -35,10 +51,26 @@ module Rdkafka
     end
 
     def with_inner
-      return if @inner.nil?
+      if @access_mutex.owned?
+        @operations_in_progress += 1
+      else
+        @access_mutex.synchronize { @operations_in_progress += 1 }
+      end
+
+      @inner.nil? ? raise(ClosedInnerError) : yield(@inner)
+    ensure
+      @decrement_mutex.synchronize { @operations_in_progress -= 1 }
+    end
 
+    def synchronize(&block)
       @access_mutex.synchronize do
-        yield @inner
+        # Wait for any commands using the inner to finish
+        # This can take a while on blocking operations like polling but is essential not to proceed
+        # with certain types of operations like resources destruction as it can cause the process
+        # to hang or crash
+        sleep(0.01) until @operations_in_progress.zero?
+
+        with_inner(&block)
       end
     end
 
@@ -53,31 +85,31 @@ module Rdkafka
     def close(object_id=nil)
       return if closed?
 
-      @access_mutex.lock
+      synchronize do
+        # Indicate to the outside world that we are closing
+        @closing = true
 
-      # Indicate to the outside world that we are closing
-      @closing = true
+        if @polling_thread
+          # Indicate to polling thread that we're closing
+          @polling_thread[:closing] = true
 
-      if @polling_thread
-        # Indicate to polling thread that we're closing
-        @polling_thread[:closing] = true
-
-        # Wait for the polling thread to finish up,
-        # this can be aborted in practice if this
-        # code runs from a finalizer.
-        @polling_thread.join
-      end
+          # Wait for the polling thread to finish up,
+          # this can be aborted in practice if this
+          # code runs from a finalizer.
+          @polling_thread.join
+        end
 
-      # Destroy the client after locking both mutexes
-      @poll_mutex.lock
+        # Destroy the client after locking both mutexes
+        @poll_mutex.lock
 
-      # This check prevents a race condition, where we would enter the close in two threads
-      # and after unlocking the primary one that hold the lock but finished, ours would be unlocked
-      # and would continue to run, trying to destroy inner twice
-      return unless @inner
+        # This check prevents a race condition, where we would enter the close in two threads
+        # and after unlocking the primary one that hold the lock but finished, ours would be unlocked
+        # and would continue to run, trying to destroy inner twice
+        return unless @inner
 
-      Rdkafka::Bindings.rd_kafka_destroy(@inner)
-      @inner = nil
+        Rdkafka::Bindings.rd_kafka_destroy(@inner)
+        @inner = nil
+      end
     end
   end
 end

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.13.0.beta.7"
+  VERSION = "0.13.0"
   LIBRDKAFKA_VERSION = "2.0.2"
   LIBRDKAFKA_SOURCE_SHA256 = "f321bcb1e015a34114c83cf1aa7b99ee260236aab096b85c003170c90a47ca9d"
 end

data/spec/rdkafka/consumer_spec.rb

@@ -286,6 +286,29 @@ describe Rdkafka::Consumer do
         consumer.poll(100)
       }.to raise_error(Rdkafka::ClosedConsumerError, /poll/)
     end
+
+    context 'when there are outgoing operations in other threads' do
+      it 'should wait and not crash' do
+        times = []
+
+        # Run a long running poll
+        thread = Thread.new do
+          times << Time.now
+          consumer.subscribe("empty_test_topic")
+          times << Time.now
+          consumer.poll(1_000)
+          times << Time.now
+        end
+
+        # Make sure it starts before we close
+        sleep(0.1)
+        consumer.close
+        close_time = Time.now
+        thread.join
+
+        times.each { |op_time| expect(op_time).to be < close_time }
+      end
+    end
   end
 
   describe "#commit, #committed and #store_offset" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.13.0.beta.7
+  version: 0.13.0
 platform: ruby
 authors:
 - Thijs Cadier
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-06-12 00:00:00.000000000 Z
+date: 2023-07-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -216,9 +216,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.4.1
 signing_key: