rdkafka 0.13.0.beta.2 → 0.13.0.beta.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eac016dc712d56e178c77a62f8af4e422319a783e80acbd7528453bfca6ca260
-  data.tar.gz: 8a4ed8df1339e0e74c96b259431012465c499ae9e9c8c6db8ff751497e1e5c43
+  metadata.gz: 5b5ba8c133537fc31e3fb5f00d230816b92d8c8f63c8d94b39f80d5d7fb22918
+  data.tar.gz: df7679de1c6724585ac5c0e636b6a239ae9d6e859496a855b2596c45e07099ca
 SHA512:
-  metadata.gz: e3167de231aaabdfa512cabef13a4ffaabcc94757a2b7d0f360ed27a9005a40da5bbb262dffc49bda03d11557eef63dcf6b9c010fb944cf41fcb06bb7531c3ee
-  data.tar.gz: 8db04d46686505a9b1fef79c8544f3c27056ae5ff9e9eede9343d4f5cef4c63c844b9798011a5339ebbdf2cab883d1c3fd5df61842ea90c4862fa0d03dd88610
+  metadata.gz: 6d1b685394205c9beaf388e40165c028dda9ef3a48ed430e9ccb24e320facadf462215b36e82a62f82cfaf61b1d6040936c352db9bb21371cb4f855d977f9c70
+  data.tar.gz: 65822e65cf3ac6297fec721756cba360f68d3533e69d76ec7208d563d39f0c659a5ac34ef21f633ad0053b9db0cc1cba887d9b4ad75b1083f2ceaf564258c722

data/.semaphore/semaphore.yml

@@ -20,8 +20,8 @@ blocks:
           commands:
             - sem-version ruby $RUBY_VERSION
             - checkout
+            - docker-compose up -d --no-recreate
             - bundle install --path vendor/bundle
             - cd ext && bundle exec rake && cd ..
-            - docker-compose up -d --no-recreate
             - ulimit -c unlimited
             - valgrind -v bundle exec rspec

data/CHANGELOG.md

@@ -7,6 +7,10 @@
 * Fix documented type for DeliveryReport#error (jimmydo)
 * Bump librdkafka to 1.9.2 (thijsc)
 * Use finalizers to cleanly exit producer and admin (thijsc)
+* Lock access to the native kafka client (thijsc)
+- Fix potential race condition in multi-threaded producer (mensfeld)
+- Fix leaking FFI resources in specs (mensfeld)
+- Improve specs stability (mensfeld)
 
 # 0.12.0
 * Bumps librdkafka to 1.9.0

data/README.md

@@ -23,6 +23,19 @@ The most important pieces of a Kafka client are implemented. We're
 working towards feature completeness, you can track that here:
 https://github.com/appsignal/rdkafka-ruby/milestone/1
 
+## Table of content
+
+- [Installation](#installation)
+- [Usage](#usage)
+  * [Consuming messages](#consuming-messages)
+  * [Producing messages](#producing-messages)
+- [Higher level libraries](#higher-level-libraries)
+  * [Message processing frameworks](#message-processing-frameworks)
+  * [Message publishing libraries](#message-publishing-libraries)
+- [Development](#development)
+- [Example](#example)
+
+
 ## Installation
 
 This gem downloads and compiles librdkafka when it is installed. If you
@@ -77,6 +90,19 @@ Note that creating a producer consumes some resources that will not be
 released until it `#close` is explicitly called, so be sure to call
 `Config#producer` only as necessary.
 
+## Higher level libraries
+
+Currently, there are two actively developed frameworks based on rdkafka-ruby, that provide higher level API that can be used to work with Kafka messages and one library for publishing messages.
+
+### Message processing frameworks
+
+* [Karafka](https://github.com/karafka/karafka) - Ruby and Rails efficient Kafka processing framework.
+* [Racecar](https://github.com/zendesk/racecar) - A simple framework for Kafka consumers in Ruby 
+
+### Message publishing libraries
+
+* [WaterDrop](https://github.com/karafka/waterdrop) – Standalone Karafka library for producing Kafka messages.
+
 ## Development
 
 A Docker Compose file is included to run Kafka and Zookeeper. To run

data/lib/rdkafka/admin.rb

@@ -18,11 +18,16 @@ module Rdkafka
 
     # Close this admin instance
     def close
+      return if closed?
       ObjectSpace.undefine_finalizer(self)
-
       @native_kafka.close
     end
 
+    # Whether this admin has closed
+    def closed?
+      @native_kafka.closed?
+    end
+
     # Create a topic with the given partition count and replication factor
     #
     # @raise [ConfigError] When the partition count or replication factor are out of valid range
@@ -62,7 +67,9 @@ module Rdkafka
       topics_array_ptr.write_array_of_pointer(pointer_array)
 
       # Get a pointer to the queue that our request will be enqueued on
-      queue_ptr = Rdkafka::Bindings.rd_kafka_queue_get_background(@native_kafka.inner)
+      queue_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
+      end
       if queue_ptr.null?
         Rdkafka::Bindings.rd_kafka_NewTopic_destroy(new_topic_ptr)
         raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
@@ -73,17 +80,21 @@ module Rdkafka
       create_topic_handle[:pending] = true
       create_topic_handle[:response] = -1
       CreateTopicHandle.register(create_topic_handle)
-      admin_options_ptr = Rdkafka::Bindings.rd_kafka_AdminOptions_new(@native_kafka.inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATETOPICS)
+      admin_options_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATETOPICS)
+      end
       Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, create_topic_handle.to_ptr)
 
       begin
-        Rdkafka::Bindings.rd_kafka_CreateTopics(
-          @native_kafka.inner,
-          topics_array_ptr,
-          1,
-          admin_options_ptr,
-          queue_ptr
-        )
+        @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_CreateTopics(
+            inner,
+            topics_array_ptr,
+            1,
+            admin_options_ptr,
+            queue_ptr
+          )
+        end
       rescue Exception
         CreateTopicHandle.remove(create_topic_handle.to_ptr.address)
         raise
@@ -113,7 +124,9 @@ module Rdkafka
       topics_array_ptr.write_array_of_pointer(pointer_array)
 
       # Get a pointer to the queue that our request will be enqueued on
-      queue_ptr = Rdkafka::Bindings.rd_kafka_queue_get_background(@native_kafka.inner)
+      queue_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
+      end
       if queue_ptr.null?
         Rdkafka::Bindings.rd_kafka_DeleteTopic_destroy(delete_topic_ptr)
         raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
@@ -124,17 +137,21 @@ module Rdkafka
       delete_topic_handle[:pending] = true
       delete_topic_handle[:response] = -1
       DeleteTopicHandle.register(delete_topic_handle)
-      admin_options_ptr = Rdkafka::Bindings.rd_kafka_AdminOptions_new(@native_kafka.inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DELETETOPICS)
+      admin_options_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DELETETOPICS)
+      end
       Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, delete_topic_handle.to_ptr)
 
       begin
-        Rdkafka::Bindings.rd_kafka_DeleteTopics(
-          @native_kafka.inner,
-          topics_array_ptr,
-          1,
-          admin_options_ptr,
-          queue_ptr
-        )
+        @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_DeleteTopics(
+            inner,
+            topics_array_ptr,
+            1,
+            admin_options_ptr,
+            queue_ptr
+          )
+        end
       rescue Exception
         DeleteTopicHandle.remove(delete_topic_handle.to_ptr.address)
         raise
@@ -149,7 +166,7 @@ module Rdkafka
 
     private
     def closed_admin_check(method)
-      raise Rdkafka::ClosedAdminError.new(method) if @native_kafka.closed?
+      raise Rdkafka::ClosedAdminError.new(method) if closed?
     end
   end
 end

data/lib/rdkafka/bindings.rb

@@ -41,10 +41,10 @@ module Rdkafka
 
     # Metadata
 
-    attach_function :rd_kafka_memberid, [:pointer], :string
-    attach_function :rd_kafka_clusterid, [:pointer], :string
-    attach_function :rd_kafka_metadata, [:pointer, :int, :pointer, :pointer, :int], :int
-    attach_function :rd_kafka_metadata_destroy, [:pointer], :void
+    attach_function :rd_kafka_memberid, [:pointer], :string, blocking: true
+    attach_function :rd_kafka_clusterid, [:pointer], :string, blocking: true
+    attach_function :rd_kafka_metadata, [:pointer, :int, :pointer, :pointer, :int], :int, blocking: true
+    attach_function :rd_kafka_metadata_destroy, [:pointer], :void, blocking: true
 
     # Message struct
 
@@ -169,26 +169,27 @@ module Rdkafka
     ]
 
     attach_function :rd_kafka_new, [:kafka_type, :pointer, :pointer, :int], :pointer
+
     attach_function :rd_kafka_destroy, [:pointer], :void
 
     # Consumer
 
-    attach_function :rd_kafka_subscribe, [:pointer, :pointer], :int
-    attach_function :rd_kafka_unsubscribe, [:pointer], :int
-    attach_function :rd_kafka_subscription, [:pointer, :pointer], :int
-    attach_function :rd_kafka_assign, [:pointer, :pointer], :int
-    attach_function :rd_kafka_incremental_assign, [:pointer, :pointer], :int
-    attach_function :rd_kafka_incremental_unassign, [:pointer, :pointer], :int
-    attach_function :rd_kafka_assignment, [:pointer, :pointer], :int
-    attach_function :rd_kafka_committed, [:pointer, :pointer, :int], :int
+    attach_function :rd_kafka_subscribe, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_unsubscribe, [:pointer], :int, blocking: true
+    attach_function :rd_kafka_subscription, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_assign, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_incremental_assign, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_incremental_unassign, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_assignment, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_committed, [:pointer, :pointer, :int], :int, blocking: true
     attach_function :rd_kafka_commit, [:pointer, :pointer, :bool], :int, blocking: true
-    attach_function :rd_kafka_poll_set_consumer, [:pointer], :void
+    attach_function :rd_kafka_poll_set_consumer, [:pointer], :void, blocking: true
     attach_function :rd_kafka_consumer_poll, [:pointer, :int], :pointer, blocking: true
     attach_function :rd_kafka_consumer_close, [:pointer], :void, blocking: true
-    attach_function :rd_kafka_offset_store, [:pointer, :int32, :int64], :int
-    attach_function :rd_kafka_pause_partitions, [:pointer, :pointer], :int
-    attach_function :rd_kafka_resume_partitions, [:pointer, :pointer], :int
-    attach_function :rd_kafka_seek, [:pointer, :int32, :int64, :int], :int
+    attach_function :rd_kafka_offset_store, [:pointer, :int32, :int64], :int, blocking: true
+    attach_function :rd_kafka_pause_partitions, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_resume_partitions, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_seek, [:pointer, :int32, :int64, :int], :int, blocking: true
 
     # Headers
     attach_function :rd_kafka_header_get_all, [:pointer, :size_t, :pointer, :pointer, SizePtr], :int
@@ -197,7 +198,7 @@ module Rdkafka
     # Rebalance
 
     callback :rebalance_cb_function, [:pointer, :int, :pointer, :pointer], :void
-    attach_function :rd_kafka_conf_set_rebalance_cb, [:pointer, :rebalance_cb_function], :void
+    attach_function :rd_kafka_conf_set_rebalance_cb, [:pointer, :rebalance_cb_function], :void, blocking: true
 
     RebalanceCallback = FFI::Function.new(
       :void, [:pointer, :int, :pointer, :pointer]
@@ -255,7 +256,7 @@ module Rdkafka
 
     RD_KAFKA_MSG_F_COPY = 0x2
 
-    attach_function :rd_kafka_producev, [:pointer, :varargs], :int
+    attach_function :rd_kafka_producev, [:pointer, :varargs], :int, blocking: true
     callback :delivery_cb, [:pointer, :pointer, :pointer], :void
     attach_function :rd_kafka_conf_set_dr_msg_cb, [:pointer, :delivery_cb], :void
 
@@ -282,23 +283,23 @@ module Rdkafka
     RD_KAFKA_ADMIN_OP_CREATETOPICS     = 1   # rd_kafka_admin_op_t
     RD_KAFKA_EVENT_CREATETOPICS_RESULT = 100 # rd_kafka_event_type_t
 
-    attach_function :rd_kafka_CreateTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :void
-    attach_function :rd_kafka_NewTopic_new, [:pointer, :size_t, :size_t, :pointer, :size_t], :pointer
-    attach_function :rd_kafka_NewTopic_set_config, [:pointer, :string, :string], :int32
-    attach_function :rd_kafka_NewTopic_destroy, [:pointer], :void
-    attach_function :rd_kafka_event_CreateTopics_result, [:pointer], :pointer
-    attach_function :rd_kafka_CreateTopics_result_topics, [:pointer, :pointer], :pointer
+    attach_function :rd_kafka_CreateTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
+    attach_function :rd_kafka_NewTopic_new, [:pointer, :size_t, :size_t, :pointer, :size_t], :pointer, blocking: true
+    attach_function :rd_kafka_NewTopic_set_config, [:pointer, :string, :string], :int32, blocking: true
+    attach_function :rd_kafka_NewTopic_destroy, [:pointer], :void, blocking: true
+    attach_function :rd_kafka_event_CreateTopics_result, [:pointer], :pointer, blocking: true
+    attach_function :rd_kafka_CreateTopics_result_topics, [:pointer, :pointer], :pointer, blocking: true
 
     # Delete Topics
 
     RD_KAFKA_ADMIN_OP_DELETETOPICS     = 2   # rd_kafka_admin_op_t
     RD_KAFKA_EVENT_DELETETOPICS_RESULT = 101 # rd_kafka_event_type_t
 
-    attach_function :rd_kafka_DeleteTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :int32
-    attach_function :rd_kafka_DeleteTopic_new, [:pointer], :pointer
-    attach_function :rd_kafka_DeleteTopic_destroy, [:pointer], :void
-    attach_function :rd_kafka_event_DeleteTopics_result, [:pointer], :pointer
-    attach_function :rd_kafka_DeleteTopics_result_topics, [:pointer, :pointer], :pointer
+    attach_function :rd_kafka_DeleteTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :int32, blocking: true
+    attach_function :rd_kafka_DeleteTopic_new, [:pointer], :pointer, blocking: true
+    attach_function :rd_kafka_DeleteTopic_destroy, [:pointer], :void, blocking: true
+    attach_function :rd_kafka_event_DeleteTopics_result, [:pointer], :pointer, blocking: true
+    attach_function :rd_kafka_DeleteTopics_result_topics, [:pointer, :pointer], :pointer, blocking: true
 
     # Background Queue and Callback
 

data/lib/rdkafka/config.rb

@@ -157,13 +157,14 @@ module Rdkafka
         Rdkafka::Bindings.rd_kafka_conf_set_rebalance_cb(config, Rdkafka::Bindings::RebalanceCallback)
       end
 
+      # Create native client
       kafka = native_kafka(config, :rd_kafka_consumer)
 
       # Redirect the main queue to the consumer
       Rdkafka::Bindings.rd_kafka_poll_set_consumer(kafka)
 
       # Return consumer with Kafka client
-      Rdkafka::Consumer.new(kafka)
+      Rdkafka::Consumer.new(Rdkafka::NativeKafka.new(kafka, run_polling_thread: false))
     end
 
     # Create a producer with this configuration.
@@ -181,7 +182,7 @@ module Rdkafka
       Rdkafka::Bindings.rd_kafka_conf_set_dr_msg_cb(config, Rdkafka::Callbacks::DeliveryCallbackFunction)
       # Return producer with Kafka client
       partitioner_name = self[:partitioner] || self["partitioner"]
-      Rdkafka::Producer.new(Rdkafka::NativeKafka.new(native_kafka(config, :rd_kafka_producer)), partitioner_name).tap do |producer|
+      Rdkafka::Producer.new(Rdkafka::NativeKafka.new(native_kafka(config, :rd_kafka_producer), run_polling_thread: true), partitioner_name).tap do |producer|
         opaque.producer = producer
       end
     end
@@ -196,7 +197,7 @@ module Rdkafka
       opaque = Opaque.new
       config = native_config(opaque)
       Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
-      Rdkafka::Admin.new(Rdkafka::NativeKafka.new(native_kafka(config, :rd_kafka_producer)))
+      Rdkafka::Admin.new(Rdkafka::NativeKafka.new(native_kafka(config, :rd_kafka_producer), run_polling_thread: true))
     end
 
     # Error that is returned by the underlying rdkafka error if an invalid configuration option is present.

data/lib/rdkafka/consumer.rb

@@ -26,15 +26,16 @@ module Rdkafka
     # @return [nil]
     def close
       return if closed?
-
-      Rdkafka::Bindings.rd_kafka_consumer_close(@native_kafka)
-      Rdkafka::Bindings.rd_kafka_destroy(@native_kafka)
-      @native_kafka = nil
+      ObjectSpace.undefine_finalizer(self)
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_consumer_close(inner)
+      end
+      @native_kafka.close
     end
 
     # Whether this consumer has closed
     def closed?
-      @native_kafka.nil?
+      @native_kafka.closed?
     end
 
     # Subscribe to one or more topics letting Kafka handle partition assignments.
@@ -55,7 +56,9 @@ module Rdkafka
       end
 
       # Subscribe to topic partition list and check this was successful
-      response = Rdkafka::Bindings.rd_kafka_subscribe(@native_kafka, tpl)
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_subscribe(inner, tpl)
+      end
       if response != 0
         raise Rdkafka::RdkafkaError.new(response, "Error subscribing to '#{topics.join(', ')}'")
       end
@@ -71,7 +74,9 @@ module Rdkafka
     def unsubscribe
       closed_consumer_check(__method__)
 
-      response = Rdkafka::Bindings.rd_kafka_unsubscribe(@native_kafka)
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_unsubscribe(inner)
+      end
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
       end
@@ -94,7 +99,9 @@ module Rdkafka
       tpl = list.to_native_tpl
 
       begin
-        response = Rdkafka::Bindings.rd_kafka_pause_partitions(@native_kafka, tpl)
+        response = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_pause_partitions(inner, tpl)
+        end
 
         if response != 0
           list = TopicPartitionList.from_native_tpl(tpl)
@@ -122,7 +129,9 @@ module Rdkafka
       tpl = list.to_native_tpl
 
       begin
-        response = Rdkafka::Bindings.rd_kafka_resume_partitions(@native_kafka, tpl)
+        response = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_resume_partitions(inner, tpl)
+        end
         if response != 0
           raise Rdkafka::RdkafkaError.new(response, "Error resume '#{list.to_h}'")
         end
@@ -140,7 +149,9 @@ module Rdkafka
       closed_consumer_check(__method__)
 
       ptr = FFI::MemoryPointer.new(:pointer)
-      response = Rdkafka::Bindings.rd_kafka_subscription(@native_kafka, ptr)
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_subscription(inner, ptr)
+      end
 
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
@@ -170,7 +181,9 @@ module Rdkafka
       tpl = list.to_native_tpl
 
       begin
-        response = Rdkafka::Bindings.rd_kafka_assign(@native_kafka, tpl)
+        response = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_assign(inner, tpl)
+        end
         if response != 0
           raise Rdkafka::RdkafkaError.new(response, "Error assigning '#{list.to_h}'")
         end
@@ -188,7 +201,9 @@ module Rdkafka
       closed_consumer_check(__method__)
 
       ptr = FFI::MemoryPointer.new(:pointer)
-      response = Rdkafka::Bindings.rd_kafka_assignment(@native_kafka, ptr)
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_assignment(inner, ptr)
+      end
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
       end
@@ -227,7 +242,9 @@ module Rdkafka
       tpl = list.to_native_tpl
 
       begin
-        response = Rdkafka::Bindings.rd_kafka_committed(@native_kafka, tpl, timeout_ms)
+        response = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_committed(inner, tpl, timeout_ms)
+        end
         if response != 0
           raise Rdkafka::RdkafkaError.new(response)
         end
@@ -252,14 +269,16 @@ module Rdkafka
       low = FFI::MemoryPointer.new(:int64, 1)
       high = FFI::MemoryPointer.new(:int64, 1)
 
-      response = Rdkafka::Bindings.rd_kafka_query_watermark_offsets(
-        @native_kafka,
-        topic,
-        partition,
-        low,
-        high,
-        timeout_ms,
-      )
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_query_watermark_offsets(
+          inner,
+          topic,
+          partition,
+          low,
+          high,
+          timeout_ms,
+        )
+      end
       if response != 0
         raise Rdkafka::RdkafkaError.new(response, "Error querying watermark offsets for partition #{partition} of #{topic}")
       end
@@ -307,7 +326,9 @@ module Rdkafka
     # @return [String, nil]
     def cluster_id
       closed_consumer_check(__method__)
-      Rdkafka::Bindings.rd_kafka_clusterid(@native_kafka)
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_clusterid(inner)
+      end
     end
 
     # Returns this client's broker-assigned group member id
@@ -317,7 +338,9 @@ module Rdkafka
     # @return [String, nil]
     def member_id
       closed_consumer_check(__method__)
-      Rdkafka::Bindings.rd_kafka_memberid(@native_kafka)
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_memberid(inner)
+      end
     end
 
     # Store offset of a message to be used in the next commit of this consumer
@@ -334,11 +357,13 @@ module Rdkafka
 
       # rd_kafka_offset_store is one of the few calls that does not support
       # a string as the topic, so create a native topic for it.
-      native_topic = Rdkafka::Bindings.rd_kafka_topic_new(
-        @native_kafka,
-        message.topic,
-        nil
-      )
+      native_topic = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_topic_new(
+          inner,
+          message.topic,
+          nil
+        )
+      end
       response = Rdkafka::Bindings.rd_kafka_offset_store(
         native_topic,
         message.partition,
@@ -366,11 +391,13 @@ module Rdkafka
 
       # rd_kafka_offset_store is one of the few calls that does not support
       # a string as the topic, so create a native topic for it.
-      native_topic = Rdkafka::Bindings.rd_kafka_topic_new(
-        @native_kafka,
-        message.topic,
-        nil
-      )
+      native_topic = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_topic_new(
+          inner,
+          message.topic,
+          nil
+        )
+      end
       response = Rdkafka::Bindings.rd_kafka_seek(
         native_topic,
         message.partition,
@@ -411,7 +438,9 @@ module Rdkafka
       tpl = list ? list.to_native_tpl : nil
 
       begin
-        response = Rdkafka::Bindings.rd_kafka_commit(@native_kafka, tpl, async)
+        response = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_commit(inner, tpl, async)
+        end
         if response != 0
           raise Rdkafka::RdkafkaError.new(response)
         end
@@ -430,7 +459,9 @@ module Rdkafka
     def poll(timeout_ms)
       closed_consumer_check(__method__)
 
-      message_ptr = Rdkafka::Bindings.rd_kafka_consumer_poll(@native_kafka, timeout_ms)
+      message_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_consumer_poll(inner, timeout_ms)
+      end
       if message_ptr.null?
         nil
       else
@@ -445,7 +476,7 @@ module Rdkafka
       end
     ensure
       # Clean up rdkafka message if there is one
-      if !message_ptr.nil? && !message_ptr.null?
+      if message_ptr && !message_ptr.null?
         Rdkafka::Bindings.rd_kafka_message_destroy(message_ptr)
       end
     end

data/lib/rdkafka/native_kafka.rb

@@ -4,25 +4,42 @@ module Rdkafka
   # @private
   # A wrapper around a native kafka that polls and cleanly exits
   class NativeKafka
-    def initialize(inner)
+    def initialize(inner, run_polling_thread:)
       @inner = inner
+      # Lock around external access
+      @access_mutex = Mutex.new
+      # Lock around internal polling
+      @poll_mutex = Mutex.new
 
-      # Start thread to poll client for delivery callbacks
-      @polling_thread = Thread.new do
-        loop do
-          Rdkafka::Bindings.rd_kafka_poll(inner, 250)
-          # Exit thread if closing and the poll queue is empty
-          if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(inner) == 0
-            break
+      if run_polling_thread
+        # Start thread to poll client for delivery callbacks,
+        # not used in consumer.
+        @polling_thread = Thread.new do
+          loop do
+            @poll_mutex.synchronize do
+              Rdkafka::Bindings.rd_kafka_poll(inner, 100)
+            end
+
+            # Exit thread if closing and the poll queue is empty
+            if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(inner) == 0
+              break
+            end
           end
         end
+
+        @polling_thread.abort_on_exception = true
+        @polling_thread[:closing] = false
       end
-      @polling_thread.abort_on_exception = true
-      @polling_thread[:closing] = false
+
+      @closing = false
     end
 
-    def inner
-      @inner
+    def with_inner
+      return if @inner.nil?
+
+      @access_mutex.synchronize do
+        yield @inner
+      end
     end
 
     def finalizer
@@ -30,22 +47,36 @@ module Rdkafka
     end
 
     def closed?
-      @inner.nil?
+      @closing || @inner.nil?
     end
 
     def close(object_id=nil)
       return if closed?
 
-      # Flush outstanding activity
-      Rdkafka::Bindings.rd_kafka_flush(@inner, 30 * 1000)
+      @access_mutex.lock
 
-      # Indicate to polling thread that we're closing
-      @polling_thread[:closing] = true
-      # Wait for the polling thread to finish up
-      @polling_thread.join
+      # Indicate to the outside world that we are closing
+      @closing = true
 
-      Rdkafka::Bindings.rd_kafka_destroy(@inner)
+      if @polling_thread
+        # Indicate to polling thread that we're closing
+        @polling_thread[:closing] = true
 
+        # Wait for the polling thread to finish up,
+        # this can be aborted in practice if this
+        # code runs from a finalizer.
+        @polling_thread.join
+      end
+
+      # Destroy the client after locking both mutexes
+      @poll_mutex.lock
+
+      # This check prevents a race condition, where we would enter the close in two threads
+      # and after unlocking the primary one that hold the lock but finished, ours would be unlocked
+      # and would continue to run, trying to destroy inner twice
+      return unless @inner
+
+      Rdkafka::Bindings.rd_kafka_destroy(@inner)
       @inner = nil
     end
   end

data/lib/rdkafka/producer.rb

@@ -40,11 +40,28 @@ module Rdkafka
 
     # Close this producer and wait for the internal poll queue to empty.
     def close
+      return if closed?
       ObjectSpace.undefine_finalizer(self)
-
       @native_kafka.close
     end
 
+    # Whether this producer has closed
+    def closed?
+      @native_kafka.closed?
+    end
+
+    # Wait until all outstanding producer requests are completed, with the given timeout
+    # in seconds. Call this before closing a producer to ensure delivery of all messages.
+    #
+    # @param timeout_ms [Integer] how long should we wait for flush of all messages
+    def flush(timeout_ms=5_000)
+      closed_producer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_flush(inner, timeout_ms)
+      end
+    end
+
     # Partition count for a given topic.
     # NOTE: If 'allow.auto.create.topics' is set to true in the broker, the topic will be auto-created after returning nil.
     #
@@ -53,7 +70,9 @@ module Rdkafka
     # @return partition count [Integer,nil]
     def partition_count(topic)
       closed_producer_check(__method__)
-      Rdkafka::Metadata.new(@native_kafka.inner, topic).topics&.first[:partition_count]
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Metadata.new(inner, topic).topics&.first[:partition_count]
+      end
     end
 
     # Produces a message to a Kafka topic. The message is added to rdkafka's queue, call {DeliveryHandle#wait wait} on the returned delivery handle to make sure it is delivered.
@@ -144,10 +163,12 @@ module Rdkafka
       args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_END
 
       # Produce the message
-      response = Rdkafka::Bindings.rd_kafka_producev(
-        @native_kafka.inner,
-        *args
-      )
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_producev(
+          inner,
+          *args
+        )
+      end
 
       # Raise error if the produce call was not successful
       if response != 0
@@ -173,7 +194,7 @@ module Rdkafka
 
     private
     def closed_producer_check(method)
-      raise Rdkafka::ClosedProducerError.new(method) if @native_kafka.closed?
+      raise Rdkafka::ClosedProducerError.new(method) if closed?
     end
   end
 end

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.13.0.beta.2"
+  VERSION = "0.13.0.beta.5"
   LIBRDKAFKA_VERSION = "1.9.2"
   LIBRDKAFKA_SOURCE_SHA256 = "3fba157a9f80a0889c982acdd44608be8a46142270a389008b22d921be1198ad"
 end

data/spec/rdkafka/config_spec.rb

@@ -151,22 +151,23 @@ describe Rdkafka::Config do
     end
 
     it "allows string partitioner key" do
-      expect(Rdkafka::Producer).to receive(:new).with(kind_of(Rdkafka::NativeKafka), "murmur2")
+      expect(Rdkafka::Producer).to receive(:new).with(kind_of(Rdkafka::NativeKafka), "murmur2").and_call_original
       config = Rdkafka::Config.new("partitioner" => "murmur2")
-      config.producer
+      config.producer.close
     end
 
     it "allows symbol partitioner key" do
-      expect(Rdkafka::Producer).to receive(:new).with(kind_of(Rdkafka::NativeKafka), "murmur2")
+      expect(Rdkafka::Producer).to receive(:new).with(kind_of(Rdkafka::NativeKafka), "murmur2").and_call_original
       config = Rdkafka::Config.new(:partitioner => "murmur2")
-      config.producer
+      config.producer.close
     end
 
     it "should allow configuring zstd compression" do
       config = Rdkafka::Config.new('compression.codec' => 'zstd')
       begin
-        expect(config.producer).to be_a Rdkafka::Producer
-        config.producer.close
+        producer = config.producer
+        expect(producer).to be_a Rdkafka::Producer
+        producer.close
       rescue Rdkafka::Config::ConfigError => ex
         pending "Zstd compression not supported on this machine"
         raise ex

data/spec/rdkafka/consumer_spec.rb

@@ -55,7 +55,7 @@ describe Rdkafka::Consumer do
 
   describe "#pause and #resume" do
     context "subscription" do
-      let(:timeout) { 1000 }
+      let(:timeout) { 2000 }
 
       before { consumer.subscribe("consume_test_topic") }
       after { consumer.unsubscribe }
@@ -738,7 +738,8 @@ describe Rdkafka::Consumer do
       #
       # This is, in effect, an integration test and the subsequent specs are
       # unit tests.
-      create_topic_handle = rdkafka_config.admin.create_topic(topic_name, 1, 1)
+      admin = rdkafka_config.admin
+      create_topic_handle = admin.create_topic(topic_name, 1, 1)
       create_topic_handle.wait(max_wait_timeout: 15.0)
       consumer.subscribe(topic_name)
       produce_n 42
@@ -751,6 +752,7 @@ describe Rdkafka::Consumer do
       expect(all_yields.flatten.size).to eq 42
       expect(all_yields.size).to be > 4
       expect(all_yields.flatten.map(&:key)).to eq (0..41).map { |x| x.to_s }
+      admin.close
     end
 
     it "should batch poll results and yield arrays of messages" do
@@ -793,13 +795,15 @@ describe Rdkafka::Consumer do
     end
 
     it "should yield [] if nothing is received before the timeout" do
-      create_topic_handle = rdkafka_config.admin.create_topic(topic_name, 1, 1)
+      admin = rdkafka_config.admin
+      create_topic_handle = admin.create_topic(topic_name, 1, 1)
       create_topic_handle.wait(max_wait_timeout: 15.0)
       consumer.subscribe(topic_name)
       consumer.each_batch do |batch|
         expect(batch).to eq([])
         break
       end
+      admin.close
     end
 
     it "should yield batchs of max_items in size if messages are already fetched" do
@@ -876,6 +880,7 @@ describe Rdkafka::Consumer do
         expect(batches_yielded.first.size).to eq 2
         expect(exceptions_yielded.flatten.size).to eq 1
         expect(exceptions_yielded.flatten.first).to be_instance_of(Rdkafka::RdkafkaError)
+        consumer.close
       end
     end
 
@@ -917,6 +922,7 @@ describe Rdkafka::Consumer do
         expect(each_batch_iterations).to eq 0
         expect(batches_yielded.size).to eq 0
         expect(exceptions_yielded.size).to eq 0
+        consumer.close
       end
     end
   end

data/spec/rdkafka/native_kafka_spec.rb

@@ -8,12 +8,9 @@ describe Rdkafka::NativeKafka do
   let(:closing) { false }
   let(:thread) { double(Thread) }
 
-  subject(:client) { described_class.new(native) }
+  subject(:client) { described_class.new(native, run_polling_thread: true) }
 
   before do
-    allow(Rdkafka::Bindings).to receive(:rd_kafka_poll).with(instance_of(FFI::Pointer), 250).and_call_original
-    allow(Rdkafka::Bindings).to receive(:rd_kafka_outq_len).with(instance_of(FFI::Pointer)).and_return(0).and_call_original
-    allow(Rdkafka::Bindings).to receive(:rd_kafka_destroy)
     allow(Thread).to receive(:new).and_return(thread)
 
     allow(thread).to receive(:[]=).with(:closing, anything)
@@ -21,6 +18,8 @@ describe Rdkafka::NativeKafka do
     allow(thread).to receive(:abort_on_exception=).with(anything)
   end
 
+  after { client.close }
+
   context "defaults" do
     it "sets the thread to abort on exception" do
       expect(thread).to receive(:abort_on_exception=).with(true)
@@ -41,32 +40,12 @@ describe Rdkafka::NativeKafka do
 
       client
     end
-
-    it "polls the native with default 250ms timeout" do
-      polling_loop_expects do
-        expect(Rdkafka::Bindings).to receive(:rd_kafka_poll).with(instance_of(FFI::Pointer), 250).at_least(:once)
-      end
-    end
-
-    it "check the out queue of native client" do
-      polling_loop_expects do
-        expect(Rdkafka::Bindings).to receive(:rd_kafka_outq_len).with(native).at_least(:once)
-      end
-    end
-  end
-
-  def polling_loop_expects(&block)
-    Thread.current[:closing] = true # this forces the loop break with line #12
-
-    allow(Thread).to receive(:new).and_yield do |_|
-      block.call
-    end.and_return(thread)
-
-    client
   end
 
-  it "exposes inner client" do
-    expect(client.inner).to eq(native)
+  it "exposes the inner client" do
+    client.with_inner do |inner|
+      expect(inner).to eq(native)
+    end
   end
 
   context "when client was not yet closed (`nil`)" do
@@ -76,7 +55,7 @@ describe Rdkafka::NativeKafka do
 
     context "and attempt to close" do
       it "calls the `destroy` binding" do
-        expect(Rdkafka::Bindings).to receive(:rd_kafka_destroy).with(native)
+        expect(Rdkafka::Bindings).to receive(:rd_kafka_destroy).with(native).and_call_original
 
         client.close
       end
@@ -96,7 +75,6 @@ describe Rdkafka::NativeKafka do
       it "closes and unassign the native client" do
         client.close
 
-        expect(client.inner).to eq(nil)
         expect(client.closed?).to eq(true)
       end
     end
@@ -111,7 +89,7 @@ describe Rdkafka::NativeKafka do
 
     context "and attempt to close again" do
       it "does not call the `destroy` binding" do
-        expect(Rdkafka::Bindings).not_to receive(:rd_kafka_destroy)
+        expect(Rdkafka::Bindings).not_to receive(:rd_kafka_destroy_flags)
 
         client.close
       end
@@ -131,7 +109,6 @@ describe Rdkafka::NativeKafka do
       it "does not close and unassign the native client again" do
         client.close
 
-        expect(client.inner).to eq(nil)
         expect(client.closed?).to eq(true)
       end
     end

data/spec/rdkafka/producer_spec.rb

@@ -185,7 +185,8 @@ describe Rdkafka::Producer do
     expect(report.partition).to eq 1
     expect(report.offset).to be >= 0
 
-    # Close producer
+    # Flush and close producer
+    producer.flush
     producer.close
 
     # Consume message and verify its content
@@ -459,10 +460,10 @@ describe Rdkafka::Producer do
     # wait for and check the message in the main process.
     reader, writer = IO.pipe
 
-    fork do
+    pid = fork do
       reader.close
 
-      # Avoids sharing the socket between processes.
+      # Avoid sharing the client between processes.
       producer = rdkafka_producer_config.producer
 
       handle = producer.produce(
@@ -481,8 +482,10 @@ describe Rdkafka::Producer do
 
       writer.write(report_json)
       writer.close
+      producer.flush
       producer.close
     end
+    Process.wait(pid)
 
     writer.close
     report_hash = JSON.parse(reader.read)

data/spec/spec_helper.rb

@@ -73,7 +73,7 @@ def new_native_topic(topic_name="topic_name", native_client: )
 end
 
 def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, consumer: nil)
-  new_consumer = !!consumer
+  new_consumer = consumer.nil?
   consumer ||= rdkafka_consumer_config.consumer
   consumer.subscribe(topic)
   timeout = Time.now.to_i + timeout_in_seconds

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.13.0.beta.2
+  version: 0.13.0.beta.5
 platform: ruby
 authors:
 - Thijs Cadier
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-10-12 00:00:00.000000000 Z
+date: 2023-04-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -139,8 +139,7 @@ dependencies:
 description: Modern Kafka client library for Ruby based on librdkafka
 email:
 - thijs@appsignal.com
-executables:
-- console
+executables: []
 extensions:
 - ext/Rakefile
 extra_rdoc_files: []
@@ -155,7 +154,6 @@ files:
 - LICENSE
 - README.md
 - Rakefile
-- bin/console
 - docker-compose.yml
 - ext/README.md
 - ext/Rakefile
@@ -222,7 +220,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubygems_version: 3.3.13
+rubygems_version: 3.4.1
 signing_key:
 specification_version: 4
 summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.

data/bin/console

@@ -1,11 +0,0 @@
-#!/usr/bin/env ruby
-
-# frozen_string_literal: true
-
-ENV["IRBRC"] = File.join(File.dirname(__FILE__), ".irbrc")
-
-require "bundler/setup"
-require "rdkafka"
-
-require "irb"
-IRB.start(__FILE__)