rdkafka 0.13.0.beta.3 → 0.13.0.beta.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9db414f84847b884bded4eb7643ead52ee664256acdde47dcf7bb5e8421b5882
-  data.tar.gz: d696bce3413d5a591542e6bb1b7191acabd5b331e4115fb7a78b6d65320874d4
+  metadata.gz: 1d9baa27220c65598729e786ff2c585cdb23db317b964fd96aee7067b5922dd1
+  data.tar.gz: 4b072016e30266c9ef8ed1ff5e88ff6caa16dd5ede48eec79e2ea22b14cbf222
 SHA512:
-  metadata.gz: cd209306c840710661108a357adbeec6a24cc1aa1a828959041a0810c25b19ddbb9b52442a69c389995361c96c075a9a572b532cc6ba97c70160d99db46f5d8d
-  data.tar.gz: 5cb50b3717ab7904d4d8afb46233f0435cf519511755ee5137ccdf638775947f23db3ff40ac1efcd0562a767e23f5a65224e8d59f3a753b33206253f9db4dfc6
+  metadata.gz: a42852f8a2bb57d28d8a55df808ebcc2cd7ca08daf1acdd139c6e8662ac81ad31da417193e5e6a6f9ed0df9f842e3842b5ad01461f1338e12deb1ca01d56830f
+  data.tar.gz: 8dde39b477b76a7afae0f9f51ec3e1942b2d4d599828bbe7acdd777c17f8fec49f4cc359e3f850db30921d0119bbe32b04c1dbef49e16f0e46d3de313cd56744

data/.semaphore/semaphore.yml

@@ -20,8 +20,8 @@ blocks:
           commands:
             - sem-version ruby $RUBY_VERSION
             - checkout
+            - docker-compose up -d --no-recreate
             - bundle install --path vendor/bundle
             - cd ext && bundle exec rake && cd ..
-            - docker-compose up -d --no-recreate
             - ulimit -c unlimited
             - valgrind -v bundle exec rspec

data/CHANGELOG.md

@@ -7,6 +7,12 @@
 * Fix documented type for DeliveryReport#error (jimmydo)
 * Bump librdkafka to 1.9.2 (thijsc)
 * Use finalizers to cleanly exit producer and admin (thijsc)
+* Lock access to the native kafka client (thijsc)
+* Fix potential race condition in multi-threaded producer (mensfeld)
+* Fix leaking FFI resources in specs (mensfeld)
+* Improve specs stability (mensfeld)
+* Make metadata request timeout configurable (mensfeld)
+* call_on_partitions_assigned and call_on_partitions_revoked only get a tpl passed in (thijsc)
 
 # 0.12.0
 * Bumps librdkafka to 1.9.0

data/README.md

@@ -23,6 +23,19 @@ The most important pieces of a Kafka client are implemented. We're
 working towards feature completeness, you can track that here:
 https://github.com/appsignal/rdkafka-ruby/milestone/1
 
+## Table of content
+
+- [Installation](#installation)
+- [Usage](#usage)
+  * [Consuming messages](#consuming-messages)
+  * [Producing messages](#producing-messages)
+- [Higher level libraries](#higher-level-libraries)
+  * [Message processing frameworks](#message-processing-frameworks)
+  * [Message publishing libraries](#message-publishing-libraries)
+- [Development](#development)
+- [Example](#example)
+
+
 ## Installation
 
 This gem downloads and compiles librdkafka when it is installed. If you
@@ -77,6 +90,19 @@ Note that creating a producer consumes some resources that will not be
 released until it `#close` is explicitly called, so be sure to call
 `Config#producer` only as necessary.
 
+## Higher level libraries
+
+Currently, there are two actively developed frameworks based on rdkafka-ruby, that provide higher level API that can be used to work with Kafka messages and one library for publishing messages.
+
+### Message processing frameworks
+
+* [Karafka](https://github.com/karafka/karafka) - Ruby and Rails efficient Kafka processing framework.
+* [Racecar](https://github.com/zendesk/racecar) - A simple framework for Kafka consumers in Ruby 
+
+### Message publishing libraries
+
+* [WaterDrop](https://github.com/karafka/waterdrop) – Standalone Karafka library for producing Kafka messages.
+
 ## Development
 
 A Docker Compose file is included to run Kafka and Zookeeper. To run

data/lib/rdkafka/admin.rb

@@ -67,7 +67,9 @@ module Rdkafka
       topics_array_ptr.write_array_of_pointer(pointer_array)
 
       # Get a pointer to the queue that our request will be enqueued on
-      queue_ptr = Rdkafka::Bindings.rd_kafka_queue_get_background(@native_kafka.inner)
+      queue_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
+      end
       if queue_ptr.null?
         Rdkafka::Bindings.rd_kafka_NewTopic_destroy(new_topic_ptr)
         raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
@@ -78,17 +80,21 @@ module Rdkafka
       create_topic_handle[:pending] = true
       create_topic_handle[:response] = -1
       CreateTopicHandle.register(create_topic_handle)
-      admin_options_ptr = Rdkafka::Bindings.rd_kafka_AdminOptions_new(@native_kafka.inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATETOPICS)
+      admin_options_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATETOPICS)
+      end
       Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, create_topic_handle.to_ptr)
 
       begin
-        Rdkafka::Bindings.rd_kafka_CreateTopics(
-          @native_kafka.inner,
-          topics_array_ptr,
-          1,
-          admin_options_ptr,
-          queue_ptr
-        )
+        @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_CreateTopics(
+            inner,
+            topics_array_ptr,
+            1,
+            admin_options_ptr,
+            queue_ptr
+          )
+        end
       rescue Exception
         CreateTopicHandle.remove(create_topic_handle.to_ptr.address)
         raise
@@ -118,7 +124,9 @@ module Rdkafka
       topics_array_ptr.write_array_of_pointer(pointer_array)
 
       # Get a pointer to the queue that our request will be enqueued on
-      queue_ptr = Rdkafka::Bindings.rd_kafka_queue_get_background(@native_kafka.inner)
+      queue_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
+      end
       if queue_ptr.null?
         Rdkafka::Bindings.rd_kafka_DeleteTopic_destroy(delete_topic_ptr)
         raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
@@ -129,17 +137,21 @@ module Rdkafka
       delete_topic_handle[:pending] = true
       delete_topic_handle[:response] = -1
       DeleteTopicHandle.register(delete_topic_handle)
-      admin_options_ptr = Rdkafka::Bindings.rd_kafka_AdminOptions_new(@native_kafka.inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DELETETOPICS)
+      admin_options_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DELETETOPICS)
+      end
       Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, delete_topic_handle.to_ptr)
 
       begin
-        Rdkafka::Bindings.rd_kafka_DeleteTopics(
-          @native_kafka.inner,
-          topics_array_ptr,
-          1,
-          admin_options_ptr,
-          queue_ptr
-        )
+        @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_DeleteTopics(
+            inner,
+            topics_array_ptr,
+            1,
+            admin_options_ptr,
+            queue_ptr
+          )
+        end
       rescue Exception
         DeleteTopicHandle.remove(delete_topic_handle.to_ptr.address)
         raise

data/lib/rdkafka/bindings.rb

@@ -41,10 +41,10 @@ module Rdkafka
 
     # Metadata
 
-    attach_function :rd_kafka_memberid, [:pointer], :string
-    attach_function :rd_kafka_clusterid, [:pointer], :string
-    attach_function :rd_kafka_metadata, [:pointer, :int, :pointer, :pointer, :int], :int
-    attach_function :rd_kafka_metadata_destroy, [:pointer], :void
+    attach_function :rd_kafka_memberid, [:pointer], :string, blocking: true
+    attach_function :rd_kafka_clusterid, [:pointer], :string, blocking: true
+    attach_function :rd_kafka_metadata, [:pointer, :int, :pointer, :pointer, :int], :int, blocking: true
+    attach_function :rd_kafka_metadata_destroy, [:pointer], :void, blocking: true
 
     # Message struct
 
@@ -170,27 +170,26 @@ module Rdkafka
 
     attach_function :rd_kafka_new, [:kafka_type, :pointer, :pointer, :int], :pointer
 
-    RD_KAFKA_DESTROY_F_IMMEDIATE = 0x4
-    attach_function :rd_kafka_destroy_flags, [:pointer, :int], :void
+    attach_function :rd_kafka_destroy, [:pointer], :void
 
     # Consumer
 
-    attach_function :rd_kafka_subscribe, [:pointer, :pointer], :int
-    attach_function :rd_kafka_unsubscribe, [:pointer], :int
-    attach_function :rd_kafka_subscription, [:pointer, :pointer], :int
-    attach_function :rd_kafka_assign, [:pointer, :pointer], :int
-    attach_function :rd_kafka_incremental_assign, [:pointer, :pointer], :int
-    attach_function :rd_kafka_incremental_unassign, [:pointer, :pointer], :int
-    attach_function :rd_kafka_assignment, [:pointer, :pointer], :int
-    attach_function :rd_kafka_committed, [:pointer, :pointer, :int], :int
+    attach_function :rd_kafka_subscribe, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_unsubscribe, [:pointer], :int, blocking: true
+    attach_function :rd_kafka_subscription, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_assign, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_incremental_assign, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_incremental_unassign, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_assignment, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_committed, [:pointer, :pointer, :int], :int, blocking: true
     attach_function :rd_kafka_commit, [:pointer, :pointer, :bool], :int, blocking: true
-    attach_function :rd_kafka_poll_set_consumer, [:pointer], :void
+    attach_function :rd_kafka_poll_set_consumer, [:pointer], :void, blocking: true
     attach_function :rd_kafka_consumer_poll, [:pointer, :int], :pointer, blocking: true
     attach_function :rd_kafka_consumer_close, [:pointer], :void, blocking: true
-    attach_function :rd_kafka_offset_store, [:pointer, :int32, :int64], :int
-    attach_function :rd_kafka_pause_partitions, [:pointer, :pointer], :int
-    attach_function :rd_kafka_resume_partitions, [:pointer, :pointer], :int
-    attach_function :rd_kafka_seek, [:pointer, :int32, :int64, :int], :int
+    attach_function :rd_kafka_offset_store, [:pointer, :int32, :int64], :int, blocking: true
+    attach_function :rd_kafka_pause_partitions, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_resume_partitions, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_seek, [:pointer, :int32, :int64, :int], :int, blocking: true
 
     # Headers
     attach_function :rd_kafka_header_get_all, [:pointer, :size_t, :pointer, :pointer, SizePtr], :int
@@ -199,7 +198,7 @@ module Rdkafka
     # Rebalance
 
     callback :rebalance_cb_function, [:pointer, :int, :pointer, :pointer], :void
-    attach_function :rd_kafka_conf_set_rebalance_cb, [:pointer, :rebalance_cb_function], :void
+    attach_function :rd_kafka_conf_set_rebalance_cb, [:pointer, :rebalance_cb_function], :void, blocking: true
 
     RebalanceCallback = FFI::Function.new(
       :void, [:pointer, :int, :pointer, :pointer]
@@ -223,14 +222,12 @@ module Rdkafka
       return unless opaque
 
       tpl = Rdkafka::Consumer::TopicPartitionList.from_native_tpl(partitions_ptr).freeze
-      consumer = Rdkafka::Consumer.new(client_ptr)
-
       begin
         case code
         when RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
-          opaque.call_on_partitions_assigned(consumer, tpl)
+          opaque.call_on_partitions_assigned(tpl)
         when RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS
-          opaque.call_on_partitions_revoked(consumer, tpl)
+          opaque.call_on_partitions_revoked(tpl)
         end
       rescue Exception => err
         Rdkafka::Config.logger.error("Unhandled exception: #{err.class} - #{err.message}")
@@ -257,7 +254,7 @@ module Rdkafka
 
     RD_KAFKA_MSG_F_COPY = 0x2
 
-    attach_function :rd_kafka_producev, [:pointer, :varargs], :int
+    attach_function :rd_kafka_producev, [:pointer, :varargs], :int, blocking: true
     callback :delivery_cb, [:pointer, :pointer, :pointer], :void
     attach_function :rd_kafka_conf_set_dr_msg_cb, [:pointer, :delivery_cb], :void
 
@@ -284,23 +281,23 @@ module Rdkafka
     RD_KAFKA_ADMIN_OP_CREATETOPICS     = 1   # rd_kafka_admin_op_t
     RD_KAFKA_EVENT_CREATETOPICS_RESULT = 100 # rd_kafka_event_type_t
 
-    attach_function :rd_kafka_CreateTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :void
-    attach_function :rd_kafka_NewTopic_new, [:pointer, :size_t, :size_t, :pointer, :size_t], :pointer
-    attach_function :rd_kafka_NewTopic_set_config, [:pointer, :string, :string], :int32
-    attach_function :rd_kafka_NewTopic_destroy, [:pointer], :void
-    attach_function :rd_kafka_event_CreateTopics_result, [:pointer], :pointer
-    attach_function :rd_kafka_CreateTopics_result_topics, [:pointer, :pointer], :pointer
+    attach_function :rd_kafka_CreateTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
+    attach_function :rd_kafka_NewTopic_new, [:pointer, :size_t, :size_t, :pointer, :size_t], :pointer, blocking: true
+    attach_function :rd_kafka_NewTopic_set_config, [:pointer, :string, :string], :int32, blocking: true
+    attach_function :rd_kafka_NewTopic_destroy, [:pointer], :void, blocking: true
+    attach_function :rd_kafka_event_CreateTopics_result, [:pointer], :pointer, blocking: true
+    attach_function :rd_kafka_CreateTopics_result_topics, [:pointer, :pointer], :pointer, blocking: true
 
     # Delete Topics
 
     RD_KAFKA_ADMIN_OP_DELETETOPICS     = 2   # rd_kafka_admin_op_t
     RD_KAFKA_EVENT_DELETETOPICS_RESULT = 101 # rd_kafka_event_type_t
 
-    attach_function :rd_kafka_DeleteTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :int32
-    attach_function :rd_kafka_DeleteTopic_new, [:pointer], :pointer
-    attach_function :rd_kafka_DeleteTopic_destroy, [:pointer], :void
-    attach_function :rd_kafka_event_DeleteTopics_result, [:pointer], :pointer
-    attach_function :rd_kafka_DeleteTopics_result_topics, [:pointer, :pointer], :pointer
+    attach_function :rd_kafka_DeleteTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :int32, blocking: true
+    attach_function :rd_kafka_DeleteTopic_new, [:pointer], :pointer, blocking: true
+    attach_function :rd_kafka_DeleteTopic_destroy, [:pointer], :void, blocking: true
+    attach_function :rd_kafka_event_DeleteTopics_result, [:pointer], :pointer, blocking: true
+    attach_function :rd_kafka_DeleteTopics_result_topics, [:pointer, :pointer], :pointer, blocking: true
 
     # Background Queue and Callback
 

data/lib/rdkafka/config.rb

@@ -285,18 +285,18 @@ module Rdkafka
       producer.call_delivery_callback(delivery_report, delivery_handle) if producer
     end
 
-    def call_on_partitions_assigned(consumer, list)
+    def call_on_partitions_assigned(list)
       return unless consumer_rebalance_listener
       return unless consumer_rebalance_listener.respond_to?(:on_partitions_assigned)
 
-      consumer_rebalance_listener.on_partitions_assigned(consumer, list)
+      consumer_rebalance_listener.on_partitions_assigned(list)
     end
 
-    def call_on_partitions_revoked(consumer, list)
+    def call_on_partitions_revoked(list)
       return unless consumer_rebalance_listener
       return unless consumer_rebalance_listener.respond_to?(:on_partitions_revoked)
 
-      consumer_rebalance_listener.on_partitions_revoked(consumer, list)
+      consumer_rebalance_listener.on_partitions_revoked(list)
     end
   end
 end

data/lib/rdkafka/consumer.rb

@@ -27,7 +27,9 @@ module Rdkafka
     def close
       return if closed?
       ObjectSpace.undefine_finalizer(self)
-      Rdkafka::Bindings.rd_kafka_consumer_close(@native_kafka.inner)
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_consumer_close(inner)
+      end
       @native_kafka.close
     end
 
@@ -54,7 +56,9 @@ module Rdkafka
       end
 
       # Subscribe to topic partition list and check this was successful
-      response = Rdkafka::Bindings.rd_kafka_subscribe(@native_kafka.inner, tpl)
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_subscribe(inner, tpl)
+      end
       if response != 0
         raise Rdkafka::RdkafkaError.new(response, "Error subscribing to '#{topics.join(', ')}'")
       end
@@ -70,7 +74,9 @@ module Rdkafka
     def unsubscribe
       closed_consumer_check(__method__)
 
-      response = Rdkafka::Bindings.rd_kafka_unsubscribe(@native_kafka.inner)
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_unsubscribe(inner)
+      end
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
       end
@@ -93,7 +99,9 @@ module Rdkafka
       tpl = list.to_native_tpl
 
       begin
-        response = Rdkafka::Bindings.rd_kafka_pause_partitions(@native_kafka.inner, tpl)
+        response = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_pause_partitions(inner, tpl)
+        end
 
         if response != 0
           list = TopicPartitionList.from_native_tpl(tpl)
@@ -121,7 +129,9 @@ module Rdkafka
       tpl = list.to_native_tpl
 
       begin
-        response = Rdkafka::Bindings.rd_kafka_resume_partitions(@native_kafka.inner, tpl)
+        response = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_resume_partitions(inner, tpl)
+        end
         if response != 0
           raise Rdkafka::RdkafkaError.new(response, "Error resume '#{list.to_h}'")
         end
@@ -139,7 +149,9 @@ module Rdkafka
       closed_consumer_check(__method__)
 
       ptr = FFI::MemoryPointer.new(:pointer)
-      response = Rdkafka::Bindings.rd_kafka_subscription(@native_kafka.inner, ptr)
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_subscription(inner, ptr)
+      end
 
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
@@ -169,7 +181,9 @@ module Rdkafka
       tpl = list.to_native_tpl
 
       begin
-        response = Rdkafka::Bindings.rd_kafka_assign(@native_kafka.inner, tpl)
+        response = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_assign(inner, tpl)
+        end
         if response != 0
           raise Rdkafka::RdkafkaError.new(response, "Error assigning '#{list.to_h}'")
         end
@@ -187,7 +201,9 @@ module Rdkafka
       closed_consumer_check(__method__)
 
       ptr = FFI::MemoryPointer.new(:pointer)
-      response = Rdkafka::Bindings.rd_kafka_assignment(@native_kafka.inner, ptr)
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_assignment(inner, ptr)
+      end
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
       end
@@ -226,7 +242,9 @@ module Rdkafka
       tpl = list.to_native_tpl
 
       begin
-        response = Rdkafka::Bindings.rd_kafka_committed(@native_kafka.inner, tpl, timeout_ms)
+        response = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_committed(inner, tpl, timeout_ms)
+        end
         if response != 0
           raise Rdkafka::RdkafkaError.new(response)
         end
@@ -251,14 +269,16 @@ module Rdkafka
       low = FFI::MemoryPointer.new(:int64, 1)
       high = FFI::MemoryPointer.new(:int64, 1)
 
-      response = Rdkafka::Bindings.rd_kafka_query_watermark_offsets(
-        @native_kafka.inner,
-        topic,
-        partition,
-        low,
-        high,
-        timeout_ms,
-      )
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_query_watermark_offsets(
+          inner,
+          topic,
+          partition,
+          low,
+          high,
+          timeout_ms,
+        )
+      end
       if response != 0
         raise Rdkafka::RdkafkaError.new(response, "Error querying watermark offsets for partition #{partition} of #{topic}")
       end
@@ -306,7 +326,9 @@ module Rdkafka
     # @return [String, nil]
     def cluster_id
       closed_consumer_check(__method__)
-      Rdkafka::Bindings.rd_kafka_clusterid(@native_kafka.inner)
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_clusterid(inner)
+      end
     end
 
     # Returns this client's broker-assigned group member id
@@ -316,7 +338,9 @@ module Rdkafka
     # @return [String, nil]
     def member_id
       closed_consumer_check(__method__)
-      Rdkafka::Bindings.rd_kafka_memberid(@native_kafka.inner)
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_memberid(inner)
+      end
     end
 
     # Store offset of a message to be used in the next commit of this consumer
@@ -333,11 +357,13 @@ module Rdkafka
 
       # rd_kafka_offset_store is one of the few calls that does not support
       # a string as the topic, so create a native topic for it.
-      native_topic = Rdkafka::Bindings.rd_kafka_topic_new(
-        @native_kafka.inner,
-        message.topic,
-        nil
-      )
+      native_topic = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_topic_new(
+          inner,
+          message.topic,
+          nil
+        )
+      end
       response = Rdkafka::Bindings.rd_kafka_offset_store(
         native_topic,
         message.partition,
@@ -365,11 +391,13 @@ module Rdkafka
 
       # rd_kafka_offset_store is one of the few calls that does not support
       # a string as the topic, so create a native topic for it.
-      native_topic = Rdkafka::Bindings.rd_kafka_topic_new(
-        @native_kafka.inner,
-        message.topic,
-        nil
-      )
+      native_topic = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_topic_new(
+          inner,
+          message.topic,
+          nil
+        )
+      end
       response = Rdkafka::Bindings.rd_kafka_seek(
         native_topic,
         message.partition,
@@ -410,7 +438,9 @@ module Rdkafka
       tpl = list ? list.to_native_tpl : nil
 
       begin
-        response = Rdkafka::Bindings.rd_kafka_commit(@native_kafka.inner, tpl, async)
+        response = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_commit(inner, tpl, async)
+        end
         if response != 0
           raise Rdkafka::RdkafkaError.new(response)
         end
@@ -429,7 +459,9 @@ module Rdkafka
     def poll(timeout_ms)
       closed_consumer_check(__method__)
 
-      message_ptr = Rdkafka::Bindings.rd_kafka_consumer_poll(@native_kafka.inner, timeout_ms)
+      message_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_consumer_poll(inner, timeout_ms)
+      end
       if message_ptr.null?
         nil
       else

data/lib/rdkafka/metadata.rb

@@ -4,7 +4,7 @@ module Rdkafka
   class Metadata
     attr_reader :brokers, :topics
 
-    def initialize(native_client, topic_name = nil)
+    def initialize(native_client, topic_name = nil, timeout_ms = 250)
       native_topic = if topic_name
         Rdkafka::Bindings.rd_kafka_topic_new(native_client, topic_name, nil)
       end
@@ -16,7 +16,7 @@ module Rdkafka
       topic_flag = topic_name.nil? ? 1 : 0
 
       # Retrieve the Metadata
-      result = Rdkafka::Bindings.rd_kafka_metadata(native_client, topic_flag, native_topic, ptr, 250)
+      result = Rdkafka::Bindings.rd_kafka_metadata(native_client, topic_flag, native_topic, ptr, timeout_ms)
 
       # Error Handling
       raise Rdkafka::RdkafkaError.new(result) unless result.zero?

data/lib/rdkafka/native_kafka.rb

@@ -6,19 +6,27 @@ module Rdkafka
   class NativeKafka
     def initialize(inner, run_polling_thread:)
       @inner = inner
+      # Lock around external access
+      @access_mutex = Mutex.new
+      # Lock around internal polling
+      @poll_mutex = Mutex.new
 
       if run_polling_thread
         # Start thread to poll client for delivery callbacks,
         # not used in consumer.
         @polling_thread = Thread.new do
           loop do
-            Rdkafka::Bindings.rd_kafka_poll(inner, 250)
+            @poll_mutex.synchronize do
+              Rdkafka::Bindings.rd_kafka_poll(inner, 100)
+            end
+
             # Exit thread if closing and the poll queue is empty
             if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(inner) == 0
               break
             end
           end
         end
+
         @polling_thread.abort_on_exception = true
         @polling_thread[:closing] = false
       end
@@ -26,8 +34,12 @@ module Rdkafka
       @closing = false
     end
 
-    def inner
-      @inner
+    def with_inner
+      return if @inner.nil?
+
+      @access_mutex.synchronize do
+        yield @inner
+      end
     end
 
     def finalizer
@@ -41,21 +53,30 @@ module Rdkafka
     def close(object_id=nil)
       return if closed?
 
+      @access_mutex.lock
+
       # Indicate to the outside world that we are closing
       @closing = true
 
       if @polling_thread
         # Indicate to polling thread that we're closing
         @polling_thread[:closing] = true
-        # Wait for the polling thread to finish up
+
+        # Wait for the polling thread to finish up,
+        # this can be aborted in practice if this
+        # code runs from a finalizer.
         @polling_thread.join
       end
 
-      # Destroy the client
-      Rdkafka::Bindings.rd_kafka_destroy_flags(
-        @inner,
-        Rdkafka::Bindings::RD_KAFKA_DESTROY_F_IMMEDIATE
-      )
+      # Destroy the client after locking both mutexes
+      @poll_mutex.lock
+
+      # This check prevents a race condition, where we would enter the close in two threads
+      # and after unlocking the primary one that hold the lock but finished, ours would be unlocked
+      # and would continue to run, trying to destroy inner twice
+      return unless @inner
+
+      Rdkafka::Bindings.rd_kafka_destroy(@inner)
       @inner = nil
     end
   end

data/lib/rdkafka/producer.rb

@@ -52,9 +52,14 @@ module Rdkafka
 
     # Wait until all outstanding producer requests are completed, with the given timeout
     # in seconds. Call this before closing a producer to ensure delivery of all messages.
+    #
+    # @param timeout_ms [Integer] how long should we wait for flush of all messages
     def flush(timeout_ms=5_000)
       closed_producer_check(__method__)
-      Rdkafka::Bindings.rd_kafka_flush(@native_kafka.inner, timeout_ms)
+
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_flush(inner, timeout_ms)
+      end
     end
 
     # Partition count for a given topic.
@@ -65,7 +70,9 @@ module Rdkafka
     # @return partition count [Integer,nil]
     def partition_count(topic)
       closed_producer_check(__method__)
-      Rdkafka::Metadata.new(@native_kafka.inner, topic).topics&.first[:partition_count]
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Metadata.new(inner, topic).topics&.first[:partition_count]
+      end
     end
 
     # Produces a message to a Kafka topic. The message is added to rdkafka's queue, call {DeliveryHandle#wait wait} on the returned delivery handle to make sure it is delivered.
@@ -156,10 +163,12 @@ module Rdkafka
       args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_END
 
       # Produce the message
-      response = Rdkafka::Bindings.rd_kafka_producev(
-        @native_kafka.inner,
-        *args
-      )
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_producev(
+          inner,
+          *args
+        )
+      end
 
       # Raise error if the produce call was not successful
       if response != 0

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.13.0.beta.3"
+  VERSION = "0.13.0.beta.6"
   LIBRDKAFKA_VERSION = "1.9.2"
   LIBRDKAFKA_SOURCE_SHA256 = "3fba157a9f80a0889c982acdd44608be8a46142270a389008b22d921be1198ad"
 end

data/spec/rdkafka/config_spec.rb

@@ -151,22 +151,23 @@ describe Rdkafka::Config do
     end
 
     it "allows string partitioner key" do
-      expect(Rdkafka::Producer).to receive(:new).with(kind_of(Rdkafka::NativeKafka), "murmur2")
+      expect(Rdkafka::Producer).to receive(:new).with(kind_of(Rdkafka::NativeKafka), "murmur2").and_call_original
       config = Rdkafka::Config.new("partitioner" => "murmur2")
-      config.producer
+      config.producer.close
     end
 
     it "allows symbol partitioner key" do
-      expect(Rdkafka::Producer).to receive(:new).with(kind_of(Rdkafka::NativeKafka), "murmur2")
+      expect(Rdkafka::Producer).to receive(:new).with(kind_of(Rdkafka::NativeKafka), "murmur2").and_call_original
       config = Rdkafka::Config.new(:partitioner => "murmur2")
-      config.producer
+      config.producer.close
     end
 
     it "should allow configuring zstd compression" do
       config = Rdkafka::Config.new('compression.codec' => 'zstd')
       begin
-        expect(config.producer).to be_a Rdkafka::Producer
-        config.producer.close
+        producer = config.producer
+        expect(producer).to be_a Rdkafka::Producer
+        producer.close
       rescue Rdkafka::Config::ConfigError => ex
         pending "Zstd compression not supported on this machine"
         raise ex

data/spec/rdkafka/consumer/message_spec.rb

@@ -28,7 +28,7 @@ describe Rdkafka::Consumer::Message do
   end
 
   after(:each) do
-    Rdkafka::Bindings.rd_kafka_destroy_flags(native_client, Rdkafka::Bindings::RD_KAFKA_DESTROY_F_IMMEDIATE)
+    Rdkafka::Bindings.rd_kafka_destroy(native_client)
   end
 
   subject { Rdkafka::Consumer::Message.new(native_message) }

data/spec/rdkafka/consumer_spec.rb

@@ -55,7 +55,7 @@ describe Rdkafka::Consumer do
 
   describe "#pause and #resume" do
     context "subscription" do
-      let(:timeout) { 1000 }
+      let(:timeout) { 2000 }
 
       before { consumer.subscribe("consume_test_topic") }
       after { consumer.unsubscribe }
@@ -738,7 +738,8 @@ describe Rdkafka::Consumer do
       #
       # This is, in effect, an integration test and the subsequent specs are
       # unit tests.
-      create_topic_handle = rdkafka_config.admin.create_topic(topic_name, 1, 1)
+      admin = rdkafka_config.admin
+      create_topic_handle = admin.create_topic(topic_name, 1, 1)
       create_topic_handle.wait(max_wait_timeout: 15.0)
       consumer.subscribe(topic_name)
       produce_n 42
@@ -751,6 +752,7 @@ describe Rdkafka::Consumer do
       expect(all_yields.flatten.size).to eq 42
       expect(all_yields.size).to be > 4
       expect(all_yields.flatten.map(&:key)).to eq (0..41).map { |x| x.to_s }
+      admin.close
     end
 
     it "should batch poll results and yield arrays of messages" do
@@ -793,13 +795,15 @@ describe Rdkafka::Consumer do
     end
 
     it "should yield [] if nothing is received before the timeout" do
-      create_topic_handle = rdkafka_config.admin.create_topic(topic_name, 1, 1)
+      admin = rdkafka_config.admin
+      create_topic_handle = admin.create_topic(topic_name, 1, 1)
       create_topic_handle.wait(max_wait_timeout: 15.0)
       consumer.subscribe(topic_name)
       consumer.each_batch do |batch|
         expect(batch).to eq([])
         break
       end
+      admin.close
     end
 
     it "should yield batchs of max_items in size if messages are already fetched" do
@@ -876,6 +880,7 @@ describe Rdkafka::Consumer do
         expect(batches_yielded.first.size).to eq 2
         expect(exceptions_yielded.flatten.size).to eq 1
         expect(exceptions_yielded.flatten.first).to be_instance_of(Rdkafka::RdkafkaError)
+        consumer.close
       end
     end
 
@@ -917,6 +922,7 @@ describe Rdkafka::Consumer do
         expect(each_batch_iterations).to eq 0
         expect(batches_yielded.size).to eq 0
         expect(exceptions_yielded.size).to eq 0
+        consumer.close
       end
     end
   end
@@ -931,11 +937,11 @@ describe Rdkafka::Consumer do
     context "with a working listener" do
       let(:listener) do
         Struct.new(:queue) do
-          def on_partitions_assigned(consumer, list)
+          def on_partitions_assigned(list)
             collect(:assign, list)
           end
 
-          def on_partitions_revoked(consumer, list)
+          def on_partitions_revoked(list)
             collect(:revoke, list)
           end
 
@@ -959,12 +965,12 @@ describe Rdkafka::Consumer do
     context "with a broken listener" do
       let(:listener) do
         Struct.new(:queue) do
-          def on_partitions_assigned(consumer, list)
+          def on_partitions_assigned(list)
             queue << :assigned
             raise 'boom'
           end
 
-          def on_partitions_revoked(consumer, list)
+          def on_partitions_revoked(list)
             queue << :revoked
             raise 'boom'
           end
@@ -1031,11 +1037,11 @@ describe Rdkafka::Consumer do
 
     let(:listener) do
       Struct.new(:queue) do
-        def on_partitions_assigned(consumer, list)
+        def on_partitions_assigned(list)
           collect(:assign, list)
         end
 
-        def on_partitions_revoked(consumer, list)
+        def on_partitions_revoked(list)
           collect(:revoke, list)
         end
 

data/spec/rdkafka/metadata_spec.rb

@@ -10,7 +10,7 @@ describe Rdkafka::Metadata do
 
   after do
     Rdkafka::Bindings.rd_kafka_consumer_close(native_kafka)
-    Rdkafka::Bindings.rd_kafka_destroy_flags(native_kafka, Rdkafka::Bindings::RD_KAFKA_DESTROY_F_IMMEDIATE)
+    Rdkafka::Bindings.rd_kafka_destroy(native_kafka)
   end
 
   context "passing in a topic name" do

data/spec/rdkafka/native_kafka_spec.rb

@@ -11,9 +11,6 @@ describe Rdkafka::NativeKafka do
   subject(:client) { described_class.new(native, run_polling_thread: true) }
 
   before do
-    allow(Rdkafka::Bindings).to receive(:rd_kafka_poll).with(instance_of(FFI::Pointer), 250).and_call_original
-    allow(Rdkafka::Bindings).to receive(:rd_kafka_outq_len).with(instance_of(FFI::Pointer)).and_return(0).and_call_original
-    allow(Rdkafka::Bindings).to receive(:rd_kafka_destroy_flags)
     allow(Thread).to receive(:new).and_return(thread)
 
     allow(thread).to receive(:[]=).with(:closing, anything)
@@ -21,6 +18,8 @@ describe Rdkafka::NativeKafka do
     allow(thread).to receive(:abort_on_exception=).with(anything)
   end
 
+  after { client.close }
+
   context "defaults" do
     it "sets the thread to abort on exception" do
       expect(thread).to receive(:abort_on_exception=).with(true)
@@ -41,42 +40,12 @@ describe Rdkafka::NativeKafka do
 
       client
     end
-
-    it "polls the native with default 250ms timeout" do
-      polling_loop_expects do
-        expect(Rdkafka::Bindings).to receive(:rd_kafka_poll).with(instance_of(FFI::Pointer), 250).at_least(:once)
-      end
-    end
-
-    it "check the out queue of native client" do
-      polling_loop_expects do
-        expect(Rdkafka::Bindings).to receive(:rd_kafka_outq_len).with(native).at_least(:once)
-      end
-    end
-
-    context "if not enabled" do
-      subject(:client) { described_class.new(native, run_polling_thread: false) }
-
-      it "is not created" do
-        expect(Thread).not_to receive(:new)
-
-        client
-      end
-    end
-  end
-
-  def polling_loop_expects(&block)
-    Thread.current[:closing] = true # this forces the loop break with line #12
-
-    allow(Thread).to receive(:new).and_yield do |_|
-      block.call
-    end.and_return(thread)
-
-    client
   end
 
-  it "exposes inner client" do
-    expect(client.inner).to eq(native)
+  it "exposes the inner client" do
+    client.with_inner do |inner|
+      expect(inner).to eq(native)
+    end
   end
 
   context "when client was not yet closed (`nil`)" do
@@ -86,7 +55,7 @@ describe Rdkafka::NativeKafka do
 
     context "and attempt to close" do
       it "calls the `destroy` binding" do
-        expect(Rdkafka::Bindings).to receive(:rd_kafka_destroy_flags).with(native, Rdkafka::Bindings::RD_KAFKA_DESTROY_F_IMMEDIATE)
+        expect(Rdkafka::Bindings).to receive(:rd_kafka_destroy).with(native).and_call_original
 
         client.close
       end
@@ -106,7 +75,6 @@ describe Rdkafka::NativeKafka do
       it "closes and unassign the native client" do
         client.close
 
-        expect(client.inner).to eq(nil)
         expect(client.closed?).to eq(true)
       end
     end
@@ -141,7 +109,6 @@ describe Rdkafka::NativeKafka do
       it "does not close and unassign the native client again" do
         client.close
 
-        expect(client.inner).to eq(nil)
         expect(client.closed?).to eq(true)
       end
     end

data/spec/spec_helper.rb

@@ -73,7 +73,7 @@ def new_native_topic(topic_name="topic_name", native_client: )
 end
 
 def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, consumer: nil)
-  new_consumer = !!consumer
+  new_consumer = consumer.nil?
   consumer ||= rdkafka_consumer_config.consumer
   consumer.subscribe(topic)
   timeout = Time.now.to_i + timeout_in_seconds

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.13.0.beta.3
+  version: 0.13.0.beta.6
 platform: ruby
 authors:
 - Thijs Cadier
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-10-25 00:00:00.000000000 Z
+date: 2023-04-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -139,8 +139,7 @@ dependencies:
 description: Modern Kafka client library for Ruby based on librdkafka
 email:
 - thijs@appsignal.com
-executables:
-- console
+executables: []
 extensions:
 - ext/Rakefile
 extra_rdoc_files: []
@@ -155,7 +154,6 @@ files:
 - LICENSE
 - README.md
 - Rakefile
-- bin/console
 - docker-compose.yml
 - ext/README.md
 - ext/Rakefile

data/bin/console

@@ -1,11 +0,0 @@
-#!/usr/bin/env ruby
-
-# frozen_string_literal: true
-
-ENV["IRBRC"] = File.join(File.dirname(__FILE__), ".irbrc")
-
-require "bundler/setup"
-require "rdkafka"
-
-require "irb"
-IRB.start(__FILE__)