rdkafka 0.13.0.beta.6 → 0.13.0.beta.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1d9baa27220c65598729e786ff2c585cdb23db317b964fd96aee7067b5922dd1
-  data.tar.gz: 4b072016e30266c9ef8ed1ff5e88ff6caa16dd5ede48eec79e2ea22b14cbf222
+  metadata.gz: e774c8eb7ac27fd5d89e3b28e5786bfbbc1354e7a35da0d89e7e1ebc488c0b80
+  data.tar.gz: d21b5a39de5fd22e38e9d2342a984c552b317d525a67d4128a9c78a07e0b1737
 SHA512:
-  metadata.gz: a42852f8a2bb57d28d8a55df808ebcc2cd7ca08daf1acdd139c6e8662ac81ad31da417193e5e6a6f9ed0df9f842e3842b5ad01461f1338e12deb1ca01d56830f
-  data.tar.gz: 8dde39b477b76a7afae0f9f51ec3e1942b2d4d599828bbe7acdd777c17f8fec49f4cc359e3f850db30921d0119bbe32b04c1dbef49e16f0e46d3de313cd56744
+  metadata.gz: 0cb979f4cc0a693f183e817fad944f8f24f09190ccd5d6363f3df9696441ef8b76056e05286f0214443095cd8b11b2abf9e3b8223d73a43bc09583dbab9b90dc
+  data.tar.gz: dded899763cf50da00fda25dcb03974946ccc0199eaececa2f20bc043971e3bf8a14c85610258f207424f8b76c69eac1a025f1e4b30297a2eedd4937ca705dee

data/CHANGELOG.md

@@ -5,7 +5,7 @@
 * Add topic name to delivery report (maeve)
 * Allow string partitioner config (mollyegibson)
 * Fix documented type for DeliveryReport#error (jimmydo)
-* Bump librdkafka to 1.9.2 (thijsc)
+* Bump librdkafka to 2.0.2 (lmaia)
 * Use finalizers to cleanly exit producer and admin (thijsc)
 * Lock access to the native kafka client (thijsc)
 * Fix potential race condition in multi-threaded producer (mensfeld)

data/lib/rdkafka/consumer.rb

@@ -27,9 +27,11 @@ module Rdkafka
     def close
       return if closed?
       ObjectSpace.undefine_finalizer(self)
-      @native_kafka.with_inner do |inner|
+
+      @native_kafka.synchronize do |inner|
         Rdkafka::Bindings.rd_kafka_consumer_close(inner)
       end
+
       @native_kafka.close
     end
 

data/lib/rdkafka/error.rb

@@ -92,4 +92,10 @@ module Rdkafka
       super("Illegal call to #{method.to_s} on a closed admin")
     end
   end
+
+  class ClosedInnerError < BaseError
+    def initialize
+      super("Illegal call to a closed inner librdkafka instance")
+    end
+  end
 end

data/lib/rdkafka/native_kafka.rb

@@ -10,6 +10,8 @@ module Rdkafka
       @access_mutex = Mutex.new
       # Lock around internal polling
       @poll_mutex = Mutex.new
+      # counter for operations in progress using inner
+      @operations_in_progress = 0
 
       if run_polling_thread
         # Start thread to poll client for delivery callbacks,
@@ -35,10 +37,26 @@ module Rdkafka
     end
 
     def with_inner
-      return if @inner.nil?
+      if @access_mutex.owned?
+        @operations_in_progress += 1
+      else
+        @access_mutex.synchronize { @operations_in_progress += 1 }
+      end
+
+      @inner.nil? ? raise(ClosedInnerError) : yield(@inner)
+    ensure
+      @operations_in_progress -= 1
+    end
 
+    def synchronize(&block)
       @access_mutex.synchronize do
-        yield @inner
+        # Wait for any commands using the inner to finish
+        # This can take a while on blocking operations like polling but is essential not to proceed
+        # with certain types of operations like resources destruction as it can cause the process
+        # to hang or crash
+        sleep(0.01) until @operations_in_progress.zero?
+
+        with_inner(&block)
       end
     end
 
@@ -53,31 +71,31 @@ module Rdkafka
     def close(object_id=nil)
       return if closed?
 
-      @access_mutex.lock
+      synchronize do
+        # Indicate to the outside world that we are closing
+        @closing = true
 
-      # Indicate to the outside world that we are closing
-      @closing = true
+        if @polling_thread
+          # Indicate to polling thread that we're closing
+          @polling_thread[:closing] = true
 
-      if @polling_thread
-        # Indicate to polling thread that we're closing
-        @polling_thread[:closing] = true
-
-        # Wait for the polling thread to finish up,
-        # this can be aborted in practice if this
-        # code runs from a finalizer.
-        @polling_thread.join
-      end
+          # Wait for the polling thread to finish up,
+          # this can be aborted in practice if this
+          # code runs from a finalizer.
+          @polling_thread.join
+        end
 
-      # Destroy the client after locking both mutexes
-      @poll_mutex.lock
+        # Destroy the client after locking both mutexes
+        @poll_mutex.lock
 
-      # This check prevents a race condition, where we would enter the close in two threads
-      # and after unlocking the primary one that hold the lock but finished, ours would be unlocked
-      # and would continue to run, trying to destroy inner twice
-      return unless @inner
+        # This check prevents a race condition, where we would enter the close in two threads
+        # and after unlocking the primary one that hold the lock but finished, ours would be unlocked
+        # and would continue to run, trying to destroy inner twice
+        return unless @inner
 
-      Rdkafka::Bindings.rd_kafka_destroy(@inner)
-      @inner = nil
+        Rdkafka::Bindings.rd_kafka_destroy(@inner)
+        @inner = nil
+      end
     end
   end
 end

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.13.0.beta.6"
-  LIBRDKAFKA_VERSION = "1.9.2"
-  LIBRDKAFKA_SOURCE_SHA256 = "3fba157a9f80a0889c982acdd44608be8a46142270a389008b22d921be1198ad"
+  VERSION = "0.13.0.beta.8"
+  LIBRDKAFKA_VERSION = "2.0.2"
+  LIBRDKAFKA_SOURCE_SHA256 = "f321bcb1e015a34114c83cf1aa7b99ee260236aab096b85c003170c90a47ca9d"
 end

data/spec/rdkafka/consumer_spec.rb

@@ -286,6 +286,29 @@ describe Rdkafka::Consumer do
         consumer.poll(100)
       }.to raise_error(Rdkafka::ClosedConsumerError, /poll/)
     end
+
+    context 'when there are outgoing operations in other threads' do
+      it 'should wait and not crash' do
+        times = []
+
+        # Run a long running poll
+        thread = Thread.new do
+          times << Time.now
+          consumer.subscribe("empty_test_topic")
+          times << Time.now
+          consumer.poll(1_000)
+          times << Time.now
+        end
+
+        # Make sure it starts before we close
+        sleep(0.1)
+        consumer.close
+        close_time = Time.now
+        thread.join
+
+        times.each { |op_time| expect(op_time).to be < close_time }
+      end
+    end
   end
 
   describe "#commit, #committed and #store_offset" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.13.0.beta.6
+  version: 0.13.0.beta.8
 platform: ruby
 authors:
 - Thijs Cadier
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-04-28 00:00:00.000000000 Z
+date: 2023-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -220,7 +220,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubygems_version: 3.3.13
+rubygems_version: 3.4.1
 signing_key:
 specification_version: 4
 summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.