rdkafka 0.13.0.beta.1 → 0.13.0.beta.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c3c5b37efae2485950c8a7627e09dfca50bd7c1264e18ae811b85c3f9ae7d09a
-  data.tar.gz: fb106016aae053f18f53885ae176cdb9e07078c80139146b165231cdd7e490ab
+  metadata.gz: 9db414f84847b884bded4eb7643ead52ee664256acdde47dcf7bb5e8421b5882
+  data.tar.gz: d696bce3413d5a591542e6bb1b7191acabd5b331e4115fb7a78b6d65320874d4
 SHA512:
-  metadata.gz: 2aaaf70e222ad813ec88ce49078f0e643c1860ff8ce93289134d52ac8a7104681c3f66fad48083c3d78223f7f3157dd7019bd41355933141316078bb1c5fd3aa
-  data.tar.gz: a1503635d8e51589db14db327176cbae4f8be9454938e63c96f2bfdcbd258cef1f392479955797e742463cd44ab1951f7596d23adb2b5c06ebf6ff6ab1963442
+  metadata.gz: cd209306c840710661108a357adbeec6a24cc1aa1a828959041a0810c25b19ddbb9b52442a69c389995361c96c075a9a572b532cc6ba97c70160d99db46f5d8d
+  data.tar.gz: 5cb50b3717ab7904d4d8afb46233f0435cf519511755ee5137ccdf638775947f23db3ff40ac1efcd0562a767e23f5a65224e8d59f3a753b33206253f9db4dfc6

data/CHANGELOG.md

@@ -1,4 +1,7 @@
 # 0.13.0
+* Support cooperative sticky partition assignment in the rebalance callback (methodmissing)
+* Support both string and symbol header keys (ColinDKelley)
+* Handle tombstone messages properly (kgalieva)
 * Add topic name to delivery report (maeve)
 * Allow string partitioner config (mollyegibson)
 * Fix documented type for DeliveryReport#error (jimmydo)
@@ -6,7 +9,7 @@
 * Use finalizers to cleanly exit producer and admin (thijsc)
 
 # 0.12.0
-* Bump librdkafka to 1.9.0
+* Bumps librdkafka to 1.9.0
 * Fix crash on empty partition key (mensfeld)
 * Pass the delivery handle to the callback (gvisokinskas)
 

data/lib/rdkafka/admin.rb

@@ -18,11 +18,16 @@ module Rdkafka
 
     # Close this admin instance
     def close
+      return if closed?
       ObjectSpace.undefine_finalizer(self)
-
       @native_kafka.close
     end
 
+    # Whether this admin has closed
+    def closed?
+      @native_kafka.closed?
+    end
+
     # Create a topic with the given partition count and replication factor
     #
     # @raise [ConfigError] When the partition count or replication factor are out of valid range
@@ -149,7 +154,7 @@ module Rdkafka
 
     private
     def closed_admin_check(method)
-      raise Rdkafka::ClosedAdminError.new(method) if @native_kafka.closed?
+      raise Rdkafka::ClosedAdminError.new(method) if closed?
     end
   end
 end

data/lib/rdkafka/bindings.rb

@@ -17,7 +17,7 @@ module Rdkafka
       end
     end
 
-    ffi_lib File.join(File.dirname(__FILE__), "../../ext/librdkafka.#{lib_extension}")
+    ffi_lib File.join(__dir__, "../../ext/librdkafka.#{lib_extension}")
 
     RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS = -175
     RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS = -174
@@ -113,6 +113,7 @@ module Rdkafka
     attach_function :rd_kafka_conf_set_stats_cb, [:pointer, :stats_cb], :void
     callback :error_cb, [:pointer, :int, :string, :pointer], :void
     attach_function :rd_kafka_conf_set_error_cb, [:pointer, :error_cb], :void
+    attach_function :rd_kafka_rebalance_protocol, [:pointer], :string
 
     # Log queue
     attach_function :rd_kafka_set_log_queue, [:pointer, :pointer], :void
@@ -168,7 +169,9 @@ module Rdkafka
     ]
 
     attach_function :rd_kafka_new, [:kafka_type, :pointer, :pointer, :int], :pointer
-    attach_function :rd_kafka_destroy, [:pointer], :void
+
+    RD_KAFKA_DESTROY_F_IMMEDIATE = 0x4
+    attach_function :rd_kafka_destroy_flags, [:pointer, :int], :void
 
     # Consumer
 
@@ -176,6 +179,8 @@ module Rdkafka
     attach_function :rd_kafka_unsubscribe, [:pointer], :int
     attach_function :rd_kafka_subscription, [:pointer, :pointer], :int
     attach_function :rd_kafka_assign, [:pointer, :pointer], :int
+    attach_function :rd_kafka_incremental_assign, [:pointer, :pointer], :int
+    attach_function :rd_kafka_incremental_unassign, [:pointer, :pointer], :int
     attach_function :rd_kafka_assignment, [:pointer, :pointer], :int
     attach_function :rd_kafka_committed, [:pointer, :pointer, :int], :int
     attach_function :rd_kafka_commit, [:pointer, :pointer, :bool], :int, blocking: true
@@ -201,9 +206,17 @@ module Rdkafka
     ) do |client_ptr, code, partitions_ptr, opaque_ptr|
       case code
       when RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
-        Rdkafka::Bindings.rd_kafka_assign(client_ptr, partitions_ptr)
+        if Rdkafka::Bindings.rd_kafka_rebalance_protocol(client_ptr) == "COOPERATIVE"
+          Rdkafka::Bindings.rd_kafka_incremental_assign(client_ptr, partitions_ptr)
+        else
+          Rdkafka::Bindings.rd_kafka_assign(client_ptr, partitions_ptr)
+        end
       else # RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS or errors
-        Rdkafka::Bindings.rd_kafka_assign(client_ptr, FFI::Pointer::NULL)
+        if Rdkafka::Bindings.rd_kafka_rebalance_protocol(client_ptr) == "COOPERATIVE"
+          Rdkafka::Bindings.rd_kafka_incremental_unassign(client_ptr, partitions_ptr)
+        else
+          Rdkafka::Bindings.rd_kafka_assign(client_ptr, FFI::Pointer::NULL)
+        end
       end
 
       opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]

data/lib/rdkafka/config.rb

@@ -157,13 +157,14 @@ module Rdkafka
         Rdkafka::Bindings.rd_kafka_conf_set_rebalance_cb(config, Rdkafka::Bindings::RebalanceCallback)
       end
 
+      # Create native client
       kafka = native_kafka(config, :rd_kafka_consumer)
 
       # Redirect the main queue to the consumer
       Rdkafka::Bindings.rd_kafka_poll_set_consumer(kafka)
 
       # Return consumer with Kafka client
-      Rdkafka::Consumer.new(kafka)
+      Rdkafka::Consumer.new(Rdkafka::NativeKafka.new(kafka, run_polling_thread: false))
     end
 
     # Create a producer with this configuration.
@@ -181,7 +182,7 @@ module Rdkafka
       Rdkafka::Bindings.rd_kafka_conf_set_dr_msg_cb(config, Rdkafka::Callbacks::DeliveryCallbackFunction)
       # Return producer with Kafka client
       partitioner_name = self[:partitioner] || self["partitioner"]
-      Rdkafka::Producer.new(Rdkafka::NativeKafka.new(native_kafka(config, :rd_kafka_producer)), partitioner_name).tap do |producer|
+      Rdkafka::Producer.new(Rdkafka::NativeKafka.new(native_kafka(config, :rd_kafka_producer), run_polling_thread: true), partitioner_name).tap do |producer|
         opaque.producer = producer
       end
     end
@@ -196,7 +197,7 @@ module Rdkafka
       opaque = Opaque.new
       config = native_config(opaque)
       Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
-      Rdkafka::Admin.new(Rdkafka::NativeKafka.new(native_kafka(config, :rd_kafka_producer)))
+      Rdkafka::Admin.new(Rdkafka::NativeKafka.new(native_kafka(config, :rd_kafka_producer), run_polling_thread: true))
     end
 
     # Error that is returned by the underlying rdkafka error if an invalid configuration option is present.

data/lib/rdkafka/consumer/headers.rb

@@ -2,11 +2,25 @@
 
 module Rdkafka
   class Consumer
-    # A message headers
-    class Headers
-      # Reads a native kafka's message header into ruby's hash
+    # Interface to return headers for a consumer message
+    module Headers
+      class HashWithSymbolKeysTreatedLikeStrings < Hash
+        def [](key)
+          if key.is_a?(Symbol)
+            Kernel.warn("rdkafka deprecation warning: header access with Symbol key #{key.inspect} treated as a String. " \
+                        "Please change your code to use String keys to avoid this warning. Symbol keys will break in version 1.")
+            super(key.to_s)
+          else
+            super
+          end
+        end
+      end
+
+      # Reads a librdkafka native message's headers and returns them as a Ruby Hash
+      #
+      # @param [librdkakfa message] native_message
       #
-      # @return [Hash<String, String>] a message headers
+      # @return [Hash<String, String>] headers Hash for the native_message
       #
       # @raise [Rdkafka::RdkafkaError] when fail to read headers
       #
@@ -26,7 +40,8 @@ module Rdkafka
         name_ptrptr = FFI::MemoryPointer.new(:pointer)
         value_ptrptr = FFI::MemoryPointer.new(:pointer)
         size_ptr = Rdkafka::Bindings::SizePtr.new
-        headers = {}
+
+        headers = HashWithSymbolKeysTreatedLikeStrings.new
 
         idx = 0
         loop do
@@ -53,12 +68,12 @@ module Rdkafka
 
           value = value_ptr.read_string(size)
 
-          headers[name.to_sym] = value
+          headers[name] = value
 
           idx += 1
         end
 
-        headers
+        headers.freeze
       end
     end
   end

data/lib/rdkafka/consumer/message.rb

@@ -20,7 +20,7 @@ module Rdkafka
       # @return [String, nil]
       attr_reader :key
 
-      # This message's offset in it's partition
+      # This message's offset in its partition
       # @return [Integer]
       attr_reader :offset
 

data/lib/rdkafka/consumer.rb

@@ -26,15 +26,14 @@ module Rdkafka
     # @return [nil]
     def close
       return if closed?
-
-      Rdkafka::Bindings.rd_kafka_consumer_close(@native_kafka)
-      Rdkafka::Bindings.rd_kafka_destroy(@native_kafka)
-      @native_kafka = nil
+      ObjectSpace.undefine_finalizer(self)
+      Rdkafka::Bindings.rd_kafka_consumer_close(@native_kafka.inner)
+      @native_kafka.close
     end
 
     # Whether this consumer has closed
     def closed?
-      @native_kafka.nil?
+      @native_kafka.closed?
     end
 
     # Subscribe to one or more topics letting Kafka handle partition assignments.
@@ -55,7 +54,7 @@ module Rdkafka
       end
 
       # Subscribe to topic partition list and check this was successful
-      response = Rdkafka::Bindings.rd_kafka_subscribe(@native_kafka, tpl)
+      response = Rdkafka::Bindings.rd_kafka_subscribe(@native_kafka.inner, tpl)
       if response != 0
         raise Rdkafka::RdkafkaError.new(response, "Error subscribing to '#{topics.join(', ')}'")
       end
@@ -71,7 +70,7 @@ module Rdkafka
     def unsubscribe
       closed_consumer_check(__method__)
 
-      response = Rdkafka::Bindings.rd_kafka_unsubscribe(@native_kafka)
+      response = Rdkafka::Bindings.rd_kafka_unsubscribe(@native_kafka.inner)
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
       end
@@ -94,7 +93,7 @@ module Rdkafka
       tpl = list.to_native_tpl
 
       begin
-        response = Rdkafka::Bindings.rd_kafka_pause_partitions(@native_kafka, tpl)
+        response = Rdkafka::Bindings.rd_kafka_pause_partitions(@native_kafka.inner, tpl)
 
         if response != 0
           list = TopicPartitionList.from_native_tpl(tpl)
@@ -122,7 +121,7 @@ module Rdkafka
       tpl = list.to_native_tpl
 
       begin
-        response = Rdkafka::Bindings.rd_kafka_resume_partitions(@native_kafka, tpl)
+        response = Rdkafka::Bindings.rd_kafka_resume_partitions(@native_kafka.inner, tpl)
         if response != 0
           raise Rdkafka::RdkafkaError.new(response, "Error resume '#{list.to_h}'")
         end
@@ -140,7 +139,7 @@ module Rdkafka
       closed_consumer_check(__method__)
 
       ptr = FFI::MemoryPointer.new(:pointer)
-      response = Rdkafka::Bindings.rd_kafka_subscription(@native_kafka, ptr)
+      response = Rdkafka::Bindings.rd_kafka_subscription(@native_kafka.inner, ptr)
 
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
@@ -170,7 +169,7 @@ module Rdkafka
       tpl = list.to_native_tpl
 
       begin
-        response = Rdkafka::Bindings.rd_kafka_assign(@native_kafka, tpl)
+        response = Rdkafka::Bindings.rd_kafka_assign(@native_kafka.inner, tpl)
         if response != 0
           raise Rdkafka::RdkafkaError.new(response, "Error assigning '#{list.to_h}'")
         end
@@ -188,7 +187,7 @@ module Rdkafka
       closed_consumer_check(__method__)
 
       ptr = FFI::MemoryPointer.new(:pointer)
-      response = Rdkafka::Bindings.rd_kafka_assignment(@native_kafka, ptr)
+      response = Rdkafka::Bindings.rd_kafka_assignment(@native_kafka.inner, ptr)
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
       end
@@ -227,7 +226,7 @@ module Rdkafka
       tpl = list.to_native_tpl
 
       begin
-        response = Rdkafka::Bindings.rd_kafka_committed(@native_kafka, tpl, timeout_ms)
+        response = Rdkafka::Bindings.rd_kafka_committed(@native_kafka.inner, tpl, timeout_ms)
         if response != 0
           raise Rdkafka::RdkafkaError.new(response)
         end
@@ -253,7 +252,7 @@ module Rdkafka
       high = FFI::MemoryPointer.new(:int64, 1)
 
       response = Rdkafka::Bindings.rd_kafka_query_watermark_offsets(
-        @native_kafka,
+        @native_kafka.inner,
         topic,
         partition,
         low,
@@ -307,7 +306,7 @@ module Rdkafka
     # @return [String, nil]
     def cluster_id
       closed_consumer_check(__method__)
-      Rdkafka::Bindings.rd_kafka_clusterid(@native_kafka)
+      Rdkafka::Bindings.rd_kafka_clusterid(@native_kafka.inner)
     end
 
     # Returns this client's broker-assigned group member id
@@ -317,7 +316,7 @@ module Rdkafka
     # @return [String, nil]
     def member_id
       closed_consumer_check(__method__)
-      Rdkafka::Bindings.rd_kafka_memberid(@native_kafka)
+      Rdkafka::Bindings.rd_kafka_memberid(@native_kafka.inner)
     end
 
     # Store offset of a message to be used in the next commit of this consumer
@@ -335,7 +334,7 @@ module Rdkafka
       # rd_kafka_offset_store is one of the few calls that does not support
       # a string as the topic, so create a native topic for it.
       native_topic = Rdkafka::Bindings.rd_kafka_topic_new(
-        @native_kafka,
+        @native_kafka.inner,
         message.topic,
         nil
       )
@@ -367,7 +366,7 @@ module Rdkafka
       # rd_kafka_offset_store is one of the few calls that does not support
       # a string as the topic, so create a native topic for it.
       native_topic = Rdkafka::Bindings.rd_kafka_topic_new(
-        @native_kafka,
+        @native_kafka.inner,
         message.topic,
         nil
       )
@@ -411,7 +410,7 @@ module Rdkafka
       tpl = list ? list.to_native_tpl : nil
 
       begin
-        response = Rdkafka::Bindings.rd_kafka_commit(@native_kafka, tpl, async)
+        response = Rdkafka::Bindings.rd_kafka_commit(@native_kafka.inner, tpl, async)
         if response != 0
           raise Rdkafka::RdkafkaError.new(response)
         end
@@ -430,7 +429,7 @@ module Rdkafka
     def poll(timeout_ms)
       closed_consumer_check(__method__)
 
-      message_ptr = Rdkafka::Bindings.rd_kafka_consumer_poll(@native_kafka, timeout_ms)
+      message_ptr = Rdkafka::Bindings.rd_kafka_consumer_poll(@native_kafka.inner, timeout_ms)
       if message_ptr.null?
         nil
       else
@@ -445,7 +444,7 @@ module Rdkafka
       end
     ensure
       # Clean up rdkafka message if there is one
-      if !message_ptr.nil? && !message_ptr.null?
+      if message_ptr && !message_ptr.null?
         Rdkafka::Bindings.rd_kafka_message_destroy(message_ptr)
       end
     end
@@ -550,7 +549,7 @@ module Rdkafka
         end
         if message
           slice << message
-          bytes += message.payload.bytesize
+          bytes += message.payload.bytesize if message.payload
         end
         if slice.size == max_items || bytes >= bytes_threshold || monotonic_now >= end_time - 0.001
           yield slice.dup, nil

data/lib/rdkafka/native_kafka.rb

@@ -4,21 +4,26 @@ module Rdkafka
   # @private
   # A wrapper around a native kafka that polls and cleanly exits
   class NativeKafka
-    def initialize(inner)
+    def initialize(inner, run_polling_thread:)
       @inner = inner
 
-      # Start thread to poll client for delivery callbacks
-      @polling_thread = Thread.new do
-        loop do
-          Rdkafka::Bindings.rd_kafka_poll(inner, 250)
-          # Exit thread if closing and the poll queue is empty
-          if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(inner) == 0
-            break
+      if run_polling_thread
+        # Start thread to poll client for delivery callbacks,
+        # not used in consumer.
+        @polling_thread = Thread.new do
+          loop do
+            Rdkafka::Bindings.rd_kafka_poll(inner, 250)
+            # Exit thread if closing and the poll queue is empty
+            if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(inner) == 0
+              break
+            end
           end
         end
+        @polling_thread.abort_on_exception = true
+        @polling_thread[:closing] = false
       end
-      @polling_thread.abort_on_exception = true
-      @polling_thread[:closing] = false
+
+      @closing = false
     end
 
     def inner
@@ -30,22 +35,27 @@ module Rdkafka
     end
 
     def closed?
-      @inner.nil?
+      @closing || @inner.nil?
     end
 
     def close(object_id=nil)
       return if closed?
 
-      # Flush outstanding activity
-      Rdkafka::Bindings.rd_kafka_flush(@inner, 30 * 1000)
-
-      # Indicate to polling thread that we're closing
-      @polling_thread[:closing] = true
-      # Wait for the polling thread to finish up
-      @polling_thread.join
+      # Indicate to the outside world that we are closing
+      @closing = true
 
-      Rdkafka::Bindings.rd_kafka_destroy(@inner)
+      if @polling_thread
+        # Indicate to polling thread that we're closing
+        @polling_thread[:closing] = true
+        # Wait for the polling thread to finish up
+        @polling_thread.join
+      end
 
+      # Destroy the client
+      Rdkafka::Bindings.rd_kafka_destroy_flags(
+        @inner,
+        Rdkafka::Bindings::RD_KAFKA_DESTROY_F_IMMEDIATE
+      )
       @inner = nil
     end
   end

data/lib/rdkafka/producer.rb

@@ -40,11 +40,23 @@ module Rdkafka
 
     # Close this producer and wait for the internal poll queue to empty.
     def close
+      return if closed?
       ObjectSpace.undefine_finalizer(self)
-
       @native_kafka.close
     end
 
+    # Whether this producer has closed
+    def closed?
+      @native_kafka.closed?
+    end
+
+    # Wait until all outstanding producer requests are completed, with the given timeout
+    # in seconds. Call this before closing a producer to ensure delivery of all messages.
+    def flush(timeout_ms=5_000)
+      closed_producer_check(__method__)
+      Rdkafka::Bindings.rd_kafka_flush(@native_kafka.inner, timeout_ms)
+    end
+
     # Partition count for a given topic.
     # NOTE: If 'allow.auto.create.topics' is set to true in the broker, the topic will be auto-created after returning nil.
     #
@@ -173,7 +185,7 @@ module Rdkafka
 
     private
     def closed_producer_check(method)
-      raise Rdkafka::ClosedProducerError.new(method) if @native_kafka.closed?
+      raise Rdkafka::ClosedProducerError.new(method) if closed?
     end
   end
 end

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.13.0.beta.1"
+  VERSION = "0.13.0.beta.3"
   LIBRDKAFKA_VERSION = "1.9.2"
   LIBRDKAFKA_SOURCE_SHA256 = "3fba157a9f80a0889c982acdd44608be8a46142270a389008b22d921be1198ad"
 end

data/spec/rdkafka/consumer/headers_spec.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+describe Rdkafka::Consumer::Headers do
+  let(:headers) do
+    { # Note String keys!
+      "version" => "2.1.3",
+      "type" => "String"
+    }
+  end
+  let(:native_message) { double('native message') }
+  let(:headers_ptr) { double('headers pointer') }
+
+  describe '.from_native' do
+    before do
+      expect(Rdkafka::Bindings).to receive(:rd_kafka_message_headers).with(native_message, anything) do |_, headers_ptrptr|
+        expect(headers_ptrptr).to receive(:read_pointer).and_return(headers_ptr)
+        Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+      end
+
+      expect(Rdkafka::Bindings).to \
+        receive(:rd_kafka_header_get_all)
+          .with(headers_ptr, 0, anything, anything, anything) do |_, _, name_ptrptr, value_ptrptr, size_ptr|
+              expect(name_ptrptr).to receive(:read_pointer).and_return(double("pointer 0", read_string_to_null: headers.keys[0]))
+              expect(size_ptr).to receive(:[]).with(:value).and_return(headers.keys[0].size)
+              expect(value_ptrptr).to receive(:read_pointer).and_return(double("value pointer 0", read_string: headers.values[0]))
+              Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+            end
+
+      expect(Rdkafka::Bindings).to \
+        receive(:rd_kafka_header_get_all)
+          .with(headers_ptr, 1, anything, anything, anything) do |_, _, name_ptrptr, value_ptrptr, size_ptr|
+              expect(name_ptrptr).to receive(:read_pointer).and_return(double("pointer 1", read_string_to_null: headers.keys[1]))
+              expect(size_ptr).to receive(:[]).with(:value).and_return(headers.keys[1].size)
+              expect(value_ptrptr).to receive(:read_pointer).and_return(double("value pointer 1", read_string: headers.values[1]))
+              Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+            end
+
+      expect(Rdkafka::Bindings).to \
+        receive(:rd_kafka_header_get_all)
+          .with(headers_ptr, 2, anything, anything, anything)
+          .and_return(Rdkafka::Bindings::RD_KAFKA_RESP_ERR__NOENT)
+    end
+
+    subject { described_class.from_native(native_message) }
+
+    it { is_expected.to eq(headers) }
+    it { is_expected.to be_frozen }
+
+    it 'allows String key' do
+      expect(subject['version']).to eq("2.1.3")
+    end
+
+    it 'allows Symbol key, but warns' do
+      expect(Kernel).to \
+        receive(:warn).with("rdkafka deprecation warning: header access with Symbol key :version treated as a String. " \
+                            "Please change your code to use String keys to avoid this warning. Symbol keys will break in version 1.")
+      expect(subject[:version]).to eq("2.1.3")
+    end
+  end
+end

data/spec/rdkafka/consumer/message_spec.rb

@@ -28,7 +28,7 @@ describe Rdkafka::Consumer::Message do
   end
 
   after(:each) do
-    Rdkafka::Bindings.rd_kafka_destroy(native_client)
+    Rdkafka::Bindings.rd_kafka_destroy_flags(native_client, Rdkafka::Bindings::RD_KAFKA_DESTROY_F_IMMEDIATE)
   end
 
   subject { Rdkafka::Consumer::Message.new(native_message) }

data/spec/rdkafka/consumer_spec.rb

@@ -595,7 +595,7 @@ describe Rdkafka::Consumer do
   end
 
   describe "#poll with headers" do
-    it "should return message with headers" do
+    it "should return message with headers using string keys (when produced with symbol keys)" do
       report = producer.produce(
         topic:     "consume_test_topic",
         key:       "key headers",
@@ -605,7 +605,20 @@ describe Rdkafka::Consumer do
       message = wait_for_message(topic: "consume_test_topic", consumer: consumer, delivery_report: report)
       expect(message).to be
       expect(message.key).to eq('key headers')
-      expect(message.headers).to include(foo: 'bar')
+      expect(message.headers).to include('foo' => 'bar')
+    end
+
+    it "should return message with headers using string keys (when produced with string keys)" do
+      report = producer.produce(
+        topic:     "consume_test_topic",
+        key:       "key headers",
+        headers:   { 'foo' => 'bar' }
+      ).wait
+
+      message = wait_for_message(topic: "consume_test_topic", consumer: consumer, delivery_report: report)
+      expect(message).to be
+      expect(message.key).to eq('key headers')
+      expect(message.headers).to include('foo' => 'bar')
     end
 
     it "should return message with no headers" do
@@ -700,7 +713,7 @@ describe Rdkafka::Consumer do
       n.times do |i|
         handles << producer.produce(
           topic:     topic_name,
-          payload:   Time.new.to_f.to_s,
+          payload:   i % 10 == 0 ? nil : Time.new.to_f.to_s,
           key:       i.to_s,
           partition: 0
         )
@@ -964,18 +977,6 @@ describe Rdkafka::Consumer do
         expect(listener.queue).to eq([:assigned, :revoked])
       end
     end
-
-    def notify_listener(listener)
-      # 1. subscribe and poll
-      consumer.subscribe("consume_test_topic")
-      wait_for_assignment(consumer)
-      consumer.poll(100)
-
-      # 2. unsubscribe
-      consumer.unsubscribe
-      wait_for_unassignment(consumer)
-      consumer.close
-    end
   end
 
   context "methods that should not be called after a consumer has been closed" do
@@ -1015,4 +1016,62 @@ describe Rdkafka::Consumer do
 
     expect(consumer.closed?).to eq(true)
   end
+
+  context "when the rebalance protocol is cooperative" do
+    let(:consumer) do
+      config = rdkafka_consumer_config(
+        {
+          :"partition.assignment.strategy" => "cooperative-sticky",
+          :"debug" => "consumer",
+        }
+      )
+      config.consumer_rebalance_listener = listener
+      config.consumer
+    end
+
+    let(:listener) do
+      Struct.new(:queue) do
+        def on_partitions_assigned(consumer, list)
+          collect(:assign, list)
+        end
+
+        def on_partitions_revoked(consumer, list)
+          collect(:revoke, list)
+        end
+
+        def collect(name, list)
+          partitions = list.to_h.map { |key, values| [key, values.map(&:partition)] }.flatten
+          queue << ([name] + partitions)
+        end
+      end.new([])
+    end
+
+    it "should be able to assign and unassign partitions using the cooperative partition assignment APIs" do
+      notify_listener(listener) do
+        handles = []
+        10.times do
+          handles << producer.produce(
+            topic:     "consume_test_topic",
+            payload:   "payload 1",
+            key:       "key 1",
+            partition: 0
+          )
+        end
+        handles.each(&:wait)
+
+        consumer.subscribe("consume_test_topic")
+        # Check the first 10 messages. Then close the consumer, which
+        # should break the each loop.
+        consumer.each_with_index do |message, i|
+          expect(message).to be_a Rdkafka::Consumer::Message
+          break if i == 10
+        end
+      end
+
+      expect(listener.queue).to eq([
+        [:assign, "consume_test_topic", 0, 1, 2],
+        [:revoke, "consume_test_topic", 0, 1, 2]
+      ])
+    end
+  end
 end

data/spec/rdkafka/metadata_spec.rb

@@ -10,7 +10,7 @@ describe Rdkafka::Metadata do
 
   after do
     Rdkafka::Bindings.rd_kafka_consumer_close(native_kafka)
-    Rdkafka::Bindings.rd_kafka_destroy(native_kafka)
+    Rdkafka::Bindings.rd_kafka_destroy_flags(native_kafka, Rdkafka::Bindings::RD_KAFKA_DESTROY_F_IMMEDIATE)
   end
 
   context "passing in a topic name" do

data/spec/rdkafka/native_kafka_spec.rb

@@ -8,12 +8,12 @@ describe Rdkafka::NativeKafka do
   let(:closing) { false }
   let(:thread) { double(Thread) }
 
-  subject(:client) { described_class.new(native) }
+  subject(:client) { described_class.new(native, run_polling_thread: true) }
 
   before do
     allow(Rdkafka::Bindings).to receive(:rd_kafka_poll).with(instance_of(FFI::Pointer), 250).and_call_original
     allow(Rdkafka::Bindings).to receive(:rd_kafka_outq_len).with(instance_of(FFI::Pointer)).and_return(0).and_call_original
-    allow(Rdkafka::Bindings).to receive(:rd_kafka_destroy)
+    allow(Rdkafka::Bindings).to receive(:rd_kafka_destroy_flags)
     allow(Thread).to receive(:new).and_return(thread)
 
     allow(thread).to receive(:[]=).with(:closing, anything)
@@ -53,6 +53,16 @@ describe Rdkafka::NativeKafka do
         expect(Rdkafka::Bindings).to receive(:rd_kafka_outq_len).with(native).at_least(:once)
       end
     end
+
+    context "if not enabled" do
+      subject(:client) { described_class.new(native, run_polling_thread: false) }
+
+      it "is not created" do
+        expect(Thread).not_to receive(:new)
+
+        client
+      end
+    end
   end
 
   def polling_loop_expects(&block)
@@ -76,7 +86,7 @@ describe Rdkafka::NativeKafka do
 
     context "and attempt to close" do
       it "calls the `destroy` binding" do
-        expect(Rdkafka::Bindings).to receive(:rd_kafka_destroy).with(native)
+        expect(Rdkafka::Bindings).to receive(:rd_kafka_destroy_flags).with(native, Rdkafka::Bindings::RD_KAFKA_DESTROY_F_IMMEDIATE)
 
         client.close
       end
@@ -111,7 +121,7 @@ describe Rdkafka::NativeKafka do
 
     context "and attempt to close again" do
       it "does not call the `destroy` binding" do
-        expect(Rdkafka::Bindings).not_to receive(:rd_kafka_destroy)
+        expect(Rdkafka::Bindings).not_to receive(:rd_kafka_destroy_flags)
 
         client.close
       end

data/spec/rdkafka/producer_spec.rb

@@ -9,7 +9,8 @@ describe Rdkafka::Producer do
 
   after do
     # Registry should always end up being empty
-    expect(Rdkafka::Producer::DeliveryHandle::REGISTRY).to eq({})
+    registry = Rdkafka::Producer::DeliveryHandle::REGISTRY
+    expect(registry).to be_empty, registry.inspect
     producer.close
     consumer.close
   end
@@ -184,10 +185,11 @@ describe Rdkafka::Producer do
     expect(report.partition).to eq 1
     expect(report.offset).to be >= 0
 
-    # Close producer
+    # Flush and close producer
+    producer.flush
     producer.close
 
-    # Consume message and verify it's content
+    # Consume message and verify its content
     message = wait_for_message(
       topic: "produce_test_topic",
       delivery_report: report,
@@ -211,7 +213,7 @@ describe Rdkafka::Producer do
     )
     report = handle.wait(max_wait_timeout: 5)
 
-    # Consume message and verify it's content
+    # Consume message and verify its content
     message = wait_for_message(
       topic: "produce_test_topic",
       delivery_report: report,
@@ -285,7 +287,7 @@ describe Rdkafka::Producer do
     )
     report = handle.wait(max_wait_timeout: 5)
 
-    # Consume message and verify it's content
+    # Consume message and verify its content
     message = wait_for_message(
       topic: "produce_test_topic",
       delivery_report: report,
@@ -318,7 +320,7 @@ describe Rdkafka::Producer do
       )
       report = handle.wait(max_wait_timeout: 5)
 
-      # Consume message and verify it's content
+      # Consume message and verify its content
       message = wait_for_message(
         topic: "produce_test_topic",
         delivery_report: report,
@@ -339,7 +341,7 @@ describe Rdkafka::Producer do
       )
       report = handle.wait(max_wait_timeout: 5)
 
-      # Consume message and verify it's content
+      # Consume message and verify its content
       message = wait_for_message(
         topic: "produce_test_topic",
         delivery_report: report,
@@ -359,7 +361,7 @@ describe Rdkafka::Producer do
     )
     report = handle.wait(max_wait_timeout: 5)
 
-    # Consume message and verify it's content
+    # Consume message and verify its content
     message = wait_for_message(
       topic: "produce_test_topic",
       delivery_report: report,
@@ -377,7 +379,7 @@ describe Rdkafka::Producer do
     )
     report = handle.wait(max_wait_timeout: 5)
 
-    # Consume message and verify it's content
+    # Consume message and verify its content
     message = wait_for_message(
       topic: "produce_test_topic",
       delivery_report: report,
@@ -397,7 +399,7 @@ describe Rdkafka::Producer do
     )
     report = handle.wait(max_wait_timeout: 5)
 
-    # Consume message and verify it's content
+    # Consume message and verify its content
     message = wait_for_message(
       topic: "produce_test_topic",
       delivery_report: report,
@@ -406,9 +408,9 @@ describe Rdkafka::Producer do
 
     expect(message.payload).to eq "payload headers"
     expect(message.key).to eq "key headers"
-    expect(message.headers[:foo]).to eq "bar"
-    expect(message.headers[:baz]).to eq "foobar"
-    expect(message.headers[:foobar]).to be_nil
+    expect(message.headers["foo"]).to eq "bar"
+    expect(message.headers["baz"]).to eq "foobar"
+    expect(message.headers["foobar"]).to be_nil
   end
 
   it "should produce a message with empty headers" do
@@ -420,7 +422,7 @@ describe Rdkafka::Producer do
     )
     report = handle.wait(max_wait_timeout: 5)
 
-    # Consume message and verify it's content
+    # Consume message and verify its content
     message = wait_for_message(
       topic: "produce_test_topic",
       delivery_report: report,
@@ -458,10 +460,10 @@ describe Rdkafka::Producer do
     # wait for and check the message in the main process.
     reader, writer = IO.pipe
 
-    fork do
+    pid = fork do
       reader.close
 
-      # Avoids sharing the socket between processes.
+      # Avoid sharing the client between processes.
       producer = rdkafka_producer_config.producer
 
       handle = producer.produce(
@@ -480,8 +482,10 @@ describe Rdkafka::Producer do
 
       writer.write(report_json)
       writer.close
+      producer.flush
       producer.close
     end
+    Process.wait(pid)
 
     writer.close
     report_hash = JSON.parse(reader.read)
@@ -493,7 +497,7 @@ describe Rdkafka::Producer do
 
     reader.close
 
-    # Consume message and verify it's content
+    # Consume message and verify its content
     message = wait_for_message(
       topic: "produce_test_topic",
       delivery_report: report,

data/spec/spec_helper.rb

@@ -106,6 +106,20 @@ def wait_for_unassignment(consumer)
   end
 end
 
+def notify_listener(listener, &block)
+  # 1. subscribe and poll
+  consumer.subscribe("consume_test_topic")
+  wait_for_assignment(consumer)
+  consumer.poll(100)
+
+  block.call if block
+
+  # 2. unsubscribe
+  consumer.unsubscribe
+  wait_for_unassignment(consumer)
+  consumer.close
+end
+
 RSpec.configure do |config|
   config.filter_run focus: true
   config.run_all_when_everything_filtered = true

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.13.0.beta.1
+  version: 0.13.0.beta.3
 platform: ruby
 authors:
 - Thijs Cadier
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-10-11 00:00:00.000000000 Z
+date: 2022-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -191,6 +191,7 @@ files:
 - spec/rdkafka/bindings_spec.rb
 - spec/rdkafka/callbacks_spec.rb
 - spec/rdkafka/config_spec.rb
+- spec/rdkafka/consumer/headers_spec.rb
 - spec/rdkafka/consumer/message_spec.rb
 - spec/rdkafka/consumer/partition_spec.rb
 - spec/rdkafka/consumer/topic_partition_list_spec.rb
@@ -221,7 +222,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.3.13
 signing_key:
 specification_version: 4
 summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.
@@ -237,6 +238,7 @@ test_files:
 - spec/rdkafka/bindings_spec.rb
 - spec/rdkafka/callbacks_spec.rb
 - spec/rdkafka/config_spec.rb
+- spec/rdkafka/consumer/headers_spec.rb
 - spec/rdkafka/consumer/message_spec.rb
 - spec/rdkafka/consumer/partition_spec.rb
 - spec/rdkafka/consumer/topic_partition_list_spec.rb