rdkafka 0.4.1 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a5e34bbb283ceedce620aa0533d2f45887a260e2c57fadd7cfac48a95a9fc62c
-  data.tar.gz: 5ea6f9e7d2b50b6e1a34f28e012034b84853555d312e64e0bf5ac567e1b28f37
+  metadata.gz: 277ff64f82622eddd6c31265d1b4b91b6b60bed802590bf9b4d015ea7a32ce0a
+  data.tar.gz: 027fc22c22349729bb04288f9010eda156a9d04cf9a616bb237b1fc56eb9aed5
 SHA512:
-  metadata.gz: 380b0b88c89b6926ed88fe667751779b61f1d1d6a8a495fb854c887f8049ed4197025d5cb2d5394d477096dacb4a805528bf619a55b412815859b71f1d903464
-  data.tar.gz: a316eaf6e438164c2b3ba8f095d769d48410c2271cf3653017e328531dd5c6c009bd94f3e193395ce1f84fdb2fa7b73ff0c131d4ca55a353e85f6f07aaa1c9e8
+  metadata.gz: 0cb6c97fa7ac62ead42bcd8da1d22f56b02f56e54a7671e7e2722d029839d24c7390c0fe10890530afb4e9a002e9f5fcc7e213777cf956e762d443edf466715e
+  data.tar.gz: 54865cb4b0772cffa1c91fd4e971a3e23f9f9b24959c83402460bca8b2d268e15bef8df8ea011848a57d75c4088198d02f77713ff9123647f62b7b5ece38a787

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+# 0.4.2
+* Delivery callback for producer
+* Document list param of commit method
+* Use default Homebrew openssl location if present
+* Consumer lag handles empty topics
+* End iteration in consumer when it is closed
+* Add suport for storing message offsets
+* Add missing runtime dependency to rake
+
 # 0.4.1
 * Bump librdkafka to 0.11.6
 

data/README.md

@@ -25,7 +25,7 @@ have any problems installing the gem please open an issue.
 
 ## Usage
 
-See the [documentation](http://www.rubydoc.info/github/thijsc/rdkafka-ruby/master) for full details on how to use this gem. Two quick examples:
+See the [documentation](https://www.rubydoc.info/github/appsignal/rdkafka-ruby) for full details on how to use this gem. Two quick examples:
 
 ### Consuming messages
 

data/docker-compose.yml

@@ -13,6 +13,6 @@ services:
       KAFKA_ADVERTISED_PORT: 9092
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
-      KAFKA_CREATE_TOPICS: "consume_test_topic:3:1,empty_test_topic:3:1,load_test_topic:3:1,produce_test_topic:3:1,rake_test_topic:3:1"
+      KAFKA_CREATE_TOPICS: "consume_test_topic:3:1,empty_test_topic:3:1,load_test_topic:3:1,produce_test_topic:3:1,rake_test_topic:3:1,empty_test_topic:3:1"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock

data/ext/Rakefile

@@ -21,6 +21,13 @@ task :default => :clean do
 
   # Download and compile librdkafka
   recipe = MiniPortile.new("librdkafka", Rdkafka::LIBRDKAFKA_VERSION)
+
+  # Use default homebrew openssl if we're on mac and the directory exists
+  if recipe.host.include?("darwin") && Dir.exists?("/usr/local/opt/openssl")
+    ENV["CPPFLAGS"] = "-I/usr/local/opt/openssl/include"
+    ENV["LDFLAGS"] = "-L/usr/local/opt/openssl/lib"
+  end
+
   recipe.files << {
     :url => "https://codeload.github.com/edenhill/librdkafka/tar.gz/v#{Rdkafka::LIBRDKAFKA_VERSION}",
     :sha256 => Rdkafka::LIBRDKAFKA_SOURCE_SHA256

data/lib/rdkafka/bindings.rb

@@ -39,6 +39,7 @@ module Rdkafka
     attach_function :rd_kafka_message_destroy, [:pointer], :void
     attach_function :rd_kafka_message_timestamp, [:pointer, :pointer], :int64
     attach_function :rd_kafka_topic_new, [:pointer, :string, :pointer], :pointer
+    attach_function :rd_kafka_topic_destroy, [:pointer], :pointer
     attach_function :rd_kafka_topic_name, [:pointer], :string
 
     # TopicPartition ad TopicPartitionList structs
@@ -83,6 +84,7 @@ module Rdkafka
     attach_function :rd_kafka_conf_set, [:pointer, :string, :string, :pointer, :int], :kafka_config_response
     callback :log_cb, [:pointer, :int, :string, :string], :void
     attach_function :rd_kafka_conf_set_log_cb, [:pointer, :log_cb], :void
+    attach_function :rd_kafka_conf_set_opaque, [:pointer, :pointer], :void
     callback :stats_cb, [:pointer, :string, :int, :pointer], :int
     attach_function :rd_kafka_conf_set_stats_cb, [:pointer, :stats_cb], :void
 
@@ -145,6 +147,7 @@ module Rdkafka
     attach_function :rd_kafka_poll_set_consumer, [:pointer], :void
     attach_function :rd_kafka_consumer_poll, [:pointer, :int], :pointer, blocking: true
     attach_function :rd_kafka_consumer_close, [:pointer], :void, blocking: true
+    attach_function :rd_kafka_offset_store, [:pointer, :int32, :int64], :int
 
     # Stats
 
@@ -174,10 +177,15 @@ module Rdkafka
       message = Message.new(message_ptr)
       delivery_handle_ptr_address = message[:_private].address
       if delivery_handle = Rdkafka::Producer::DeliveryHandle.remove(delivery_handle_ptr_address)
+        # Update delivery handle
         delivery_handle[:pending] = false
         delivery_handle[:response] = message[:err]
         delivery_handle[:partition] = message[:partition]
         delivery_handle[:offset] = message[:offset]
+        # Call delivery callback on opaque
+        if opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
+          opaque.call_delivery_callback(Rdkafka::Producer::DeliveryReport.new(message[:partition], message[:offset]))
+        end
       end
     end
   end

data/lib/rdkafka/config.rb

@@ -7,7 +7,10 @@ module Rdkafka
   class Config
     # @private
     @@logger = Logger.new(STDOUT)
+    # @private
     @@statistics_callback = nil
+    # @private
+    @@opaques = {}
 
     # Returns the current logger, by default this is a logger to stdout.
     #
@@ -45,6 +48,11 @@ module Rdkafka
       @@statistics_callback
     end
 
+    # @private
+    def self.opaques
+      @@opaques
+    end
+
     # Default config that can be overwritten.
     DEFAULT_CONFIG = {
       # Request api version so advanced features work
@@ -106,12 +114,16 @@ module Rdkafka
     #
     # @return [Producer] The created producer
     def producer
+      # Create opaque
+      opaque = Opaque.new
       # Create Kafka config
-      config = native_config
+      config = native_config(opaque)
       # Set callback to receive delivery reports on config
       Rdkafka::Bindings.rd_kafka_conf_set_dr_msg_cb(config, Rdkafka::Bindings::DeliveryCallback)
       # Return producer with Kafka client
-      Rdkafka::Producer.new(native_kafka(config, :rd_kafka_producer))
+      Rdkafka::Producer.new(native_kafka(config, :rd_kafka_producer)).tap do |producer|
+        opaque.producer = producer
+      end
     end
 
     # Error that is returned by the underlying rdkafka error if an invalid configuration option is present.
@@ -127,8 +139,9 @@ module Rdkafka
 
     # This method is only intented to be used to create a client,
     # using it in another way will leak memory.
-    def native_config
+    def native_config(opaque=nil)
       Rdkafka::Bindings.rd_kafka_conf_new.tap do |config|
+        # Create config
         @config_hash.merge(REQUIRED_CONFIG).each do |key, value|
           error_buffer = FFI::MemoryPointer.from_string(" " * 256)
           result = Rdkafka::Bindings.rd_kafka_conf_set(
@@ -142,10 +155,21 @@ module Rdkafka
             raise ConfigError.new(error_buffer.read_string)
           end
         end
-        # Set opaque pointer back to this config
-        #Rdkafka::Bindings.rd_kafka_conf_set_opaque(config, self)
+
+        # Set opaque pointer that's used as a proxy for callbacks
+        if opaque
+          pointer = ::FFI::Pointer.new(:pointer, opaque.object_id)
+          Rdkafka::Bindings.rd_kafka_conf_set_opaque(config, pointer)
+
+          # Store opaque with the pointer as key. We use this approach instead
+          # of trying to convert the pointer to a Ruby object because there is
+          # no risk of a segfault this way.
+          Rdkafka::Config.opaques[pointer.to_i] = opaque
+        end
+
         # Set log callback
         Rdkafka::Bindings.rd_kafka_conf_set_log_cb(config, Rdkafka::Bindings::LogCallback)
+
         # Set stats callback
         Rdkafka::Bindings.rd_kafka_conf_set_stats_cb(config, Rdkafka::Bindings::StatsCallback)
       end
@@ -176,4 +200,13 @@ module Rdkafka
       )
     end
   end
+
+  # @private
+  class Opaque
+    attr_accessor :producer
+
+    def call_delivery_callback(delivery_handle)
+      producer.call_delivery_callback(delivery_handle) if producer
+    end
+  end
 end

data/lib/rdkafka/consumer.rb

@@ -11,11 +11,13 @@ module Rdkafka
     # @private
     def initialize(native_kafka)
       @native_kafka = native_kafka
+      @closing = false
     end
 
     # Close this consumer
     # @return [nil]
     def close
+      @closing = true
       Rdkafka::Bindings.rd_kafka_consumer_close(@native_kafka)
     end
 
@@ -175,6 +177,7 @@ module Rdkafka
         # and compare to the offset in the list.
         topic_out = {}
         partitions.each do |p|
+          next if p.offset.nil?
           low, high = query_watermark_offsets(
             topic,
             p.partition,
@@ -187,8 +190,40 @@ module Rdkafka
       out
     end
 
+    # Store offset of a message to be used in the next commit of this consumer
+    #
+    # When using this `enable.auto.offset.store` should be set to `false` in the config.
+    #
+    # @param message [Rdkafka::Consumer::Message] The message which offset will be stored
+    #
+    # @raise [RdkafkaError] When storing the offset fails
+    #
+    # @return [nil]
+    def store_offset(message)
+      # rd_kafka_offset_store is one of the few calls that does not support
+      # a string as the topic, so create a native topic for it.
+      native_topic = Rdkafka::Bindings.rd_kafka_topic_new(
+        @native_kafka,
+        message.topic,
+        nil
+      )
+      response = Rdkafka::Bindings.rd_kafka_offset_store(
+        native_topic,
+        message.partition,
+        message.offset
+      )
+      if response != 0
+        raise Rdkafka::RdkafkaError.new(response)
+      end
+    ensure
+      if native_topic && !native_topic.null?
+        Rdkafka::Bindings.rd_kafka_topic_destroy(native_topic)
+      end
+    end
+
     # Commit the current offsets of this consumer
     #
+    # @param list [TopicPartitionList,nil] The topic with partitions to commit
     # @param async [Boolean] Whether to commit async or wait for the commit to finish
     #
     # @raise [RdkafkaError] When comitting fails
@@ -239,7 +274,8 @@ module Rdkafka
       end
     end
 
-    # Poll for new messages and yield for each received one
+    # Poll for new messages and yield for each received one. Iteration
+    # will end when the consumer is closed.
     #
     # @raise [RdkafkaError] When polling fails
     #
@@ -252,7 +288,11 @@ module Rdkafka
         if message
           block.call(message)
         else
-          next
+          if @closing
+            break
+          else
+            next
+          end
         end
       end
     end

data/lib/rdkafka/producer.rb

@@ -1,6 +1,9 @@
 module Rdkafka
   # A producer for Kafka messages. To create a producer set up a {Config} and call {Config#producer producer} on that.
   class Producer
+    # @private
+    @delivery_callback = nil
+
     # @private
     def initialize(native_kafka)
       @closing = false
@@ -18,6 +21,24 @@ module Rdkafka
       @polling_thread.abort_on_exception = true
     end
 
+    # Set a callback that will be called every time a message is successfully produced.
+    # The callback is called with a {DeliveryReport}
+    #
+    # @param callback [Proc] The callback
+    #
+    # @return [nil]
+    def delivery_callback=(callback)
+      raise TypeError.new("Callback has to be a proc or lambda") unless callback.is_a? Proc
+      @delivery_callback = callback
+    end
+
+    # Returns the current delivery callback, by default this is nil.
+    #
+    # @return [Proc, nil]
+    def delivery_callback
+      @delivery_callback
+    end
+
     # Close this producer and wait for the internal poll queue to empty.
     def close
       # Indicate to polling thread that we're closing
@@ -101,5 +122,10 @@ module Rdkafka
 
       delivery_handle
     end
+
+    # @private
+    def call_delivery_callback(delivery_handle)
+      @delivery_callback.call(delivery_handle) if @delivery_callback
+    end
   end
 end

data/lib/rdkafka/version.rb

@@ -1,5 +1,5 @@
 module Rdkafka
-  VERSION = "0.4.1"
+  VERSION = "0.4.2"
   LIBRDKAFKA_VERSION = "0.11.6"
   LIBRDKAFKA_SOURCE_SHA256 = '9c0afb8b53779d968225edf1e79da48a162895ad557900f75e7978f65e642032'
 end

data/rdkafka.gemspec

@@ -19,6 +19,7 @@ Gem::Specification.new do |gem|
 
   gem.add_dependency 'ffi', '~> 1.9'
   gem.add_dependency 'mini_portile2', '~> 2.1'
+  gem.add_dependency 'rake', '~> 12.3'
 
   gem.add_development_dependency 'pry', '~> 0.10'
   gem.add_development_dependency 'rspec', '~> 3.5'

data/spec/rdkafka/config_spec.rb

@@ -27,6 +27,7 @@ describe Rdkafka::Config do
           puts stats
         end
       }.not_to raise_error
+      expect(Rdkafka::Config.statistics_callback).to be_a Proc
     end
 
     it "should not accept a callback that's not a proc" do
@@ -59,6 +60,21 @@ describe Rdkafka::Config do
       }.to raise_error(Rdkafka::Config::ConfigError, "No such configuration property: \"invalid.key\"")
     end
 
+    it "should raise an error when creating a consumer with a nil key in the config" do
+      config = Rdkafka::Config.new(nil => 'value')
+      expect {
+        config.consumer
+      }.to raise_error(Rdkafka::Config::ConfigError, "No such configuration property: \"\"")
+    end
+
+    it "should treat a nil value as blank" do
+      config = Rdkafka::Config.new('security.protocol' => nil)
+      expect {
+        config.consumer
+        config.producer
+      }.to raise_error(Rdkafka::Config::ConfigError, "Configuration property \"security.protocol\" cannot be set to empty value")
+    end
+
     it "should create a producer with valid config" do
       expect(rdkafka_config.producer).to be_a Rdkafka::Producer
     end

data/spec/rdkafka/consumer_spec.rb

@@ -112,21 +112,22 @@ describe Rdkafka::Consumer do
     end
   end
 
-  describe "#commit and #committed" do
-    before do
-      # Make sure there's a stored offset
+  describe "#commit, #committed and #store_offset" do
+    # Make sure there's a stored offset
+    let!(:report) do
       report = producer.produce(
         topic:     "consume_test_topic",
         payload:   "payload 1",
         key:       "key 1",
         partition: 0
       ).wait
-      # Wait for message commits the current state,
-      # commit is therefore tested here.
-      message = wait_for_message(
+    end
+
+    let(:message) do
+      wait_for_message(
         topic: "consume_test_topic",
         delivery_report: report,
-        config: config
+        consumer: consumer
       )
     end
 
@@ -148,70 +149,107 @@ describe Rdkafka::Consumer do
       }.to raise_error TypeError
     end
 
-    it "should commit a specific topic partion list" do
-      # Make sure there are some message
-      3.times do |i|
-        producer.produce(
-          topic:     "consume_test_topic",
-          payload:   "payload 1",
-          key:       "key 1",
-          partition: i
-        ).wait
+    context "with a commited consumer" do
+      before :all do
+        # Make sure there are some message
+        producer = rdkafka_config.producer
+        handles = []
+        10.times do
+          (0..2).each do |i|
+            handles << producer.produce(
+              topic:     "consume_test_topic",
+              payload:   "payload 1",
+              key:       "key 1",
+              partition: i
+            )
+          end
+        end
+        handles.each(&:wait)
       end
 
-      list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
-        list.add_topic_and_partitions_with_offsets("consume_test_topic", {0 => 1, 1 => 1, 2 => 1})
+      before do
+        consumer.subscribe("consume_test_topic")
+        wait_for_assignment(consumer)
+        list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+          list.add_topic_and_partitions_with_offsets("consume_test_topic", 0 => 1, 1 => 1, 2 => 1)
+        end
+        consumer.commit(list)
       end
-      consumer.commit(list)
 
-      partitions = consumer.committed(list).to_h["consume_test_topic"]
-      expect(partitions[0].offset).to eq 1
-      expect(partitions[1].offset).to eq 1
-      expect(partitions[2].offset).to eq 1
-    end
+      it "should commit a specific topic partion list" do
+        list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+          list.add_topic_and_partitions_with_offsets("consume_test_topic", 0 => 1, 1 => 2, 2 => 3)
+        end
+        consumer.commit(list)
 
-    it "should raise an error when committing fails" do
-      expect(Rdkafka::Bindings).to receive(:rd_kafka_commit).and_return(20)
+        partitions = consumer.committed(list).to_h["consume_test_topic"]
+        expect(partitions[0].offset).to eq 1
+        expect(partitions[1].offset).to eq 2
+        expect(partitions[2].offset).to eq 3
+      end
 
-      expect {
-        consumer.commit
-      }.to raise_error(Rdkafka::RdkafkaError)
-    end
+      it "should raise an error when committing fails" do
+        expect(Rdkafka::Bindings).to receive(:rd_kafka_commit).and_return(20)
 
-    it "should fetch the committed offsets for the current assignment" do
-      consumer.subscribe("consume_test_topic")
-      # Wait for the assignment to be made
-      10.times do
-        break if !consumer.assignment.empty?
-        sleep 1
+        expect {
+          consumer.commit
+        }.to raise_error(Rdkafka::RdkafkaError)
       end
 
-      partitions = consumer.committed.to_h["consume_test_topic"]
-      expect(partitions).not_to be_nil
-      expect(partitions[0].offset).to be > 0
-      expect(partitions[1].offset).to be nil
-      expect(partitions[2].offset).to be nil
-    end
+      it "should fetch the committed offsets for the current assignment" do
+        partitions = consumer.committed.to_h["consume_test_topic"]
+        expect(partitions).not_to be_nil
+        expect(partitions[0].offset).to eq 1
+      end
 
-    it "should fetch the committed offsets for a specified topic partition list" do
-      list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
-        list.add_topic("consume_test_topic", [0, 1, 2])
+      it "should fetch the committed offsets for a specified topic partition list" do
+        list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+          list.add_topic("consume_test_topic", [0, 1, 2])
+        end
+        partitions = consumer.committed(list).to_h["consume_test_topic"]
+        expect(partitions).not_to be_nil
+        expect(partitions[0].offset).to eq 1
+        expect(partitions[1].offset).to eq 1
+        expect(partitions[2].offset).to eq 1
       end
-      partitions = consumer.committed(list).to_h["consume_test_topic"]
-      expect(partitions).not_to be_nil
-      expect(partitions[0].offset).to be > 0
-      expect(partitions[1].offset).to be nil
-      expect(partitions[2].offset).to be nil
-    end
 
-    it "should raise an error when getting committed fails" do
-      expect(Rdkafka::Bindings).to receive(:rd_kafka_committed).and_return(20)
-      list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
-        list.add_topic("consume_test_topic", [0, 1, 2])
+      it "should raise an error when getting committed fails" do
+        expect(Rdkafka::Bindings).to receive(:rd_kafka_committed).and_return(20)
+        list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+          list.add_topic("consume_test_topic", [0, 1, 2])
+        end
+        expect {
+          consumer.committed(list)
+        }.to raise_error Rdkafka::RdkafkaError
+      end
+
+      describe "#store_offset" do
+        before do
+          config[:'enable.auto.offset.store'] = false
+          config[:'enable.auto.commit'] = false
+          consumer.subscribe("consume_test_topic")
+          wait_for_assignment(consumer)
+        end
+
+        it "should store the offset for a message" do
+          consumer.store_offset(message)
+          consumer.commit
+
+          list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+            list.add_topic("consume_test_topic", [0, 1, 2])
+          end
+          partitions = consumer.committed(list).to_h["consume_test_topic"]
+          expect(partitions).not_to be_nil
+          expect(partitions[message.partition].offset).to eq(message.offset + 1)
+        end
+
+        it "should raise an error with invalid input" do
+          allow(message).to receive(:partition).and_return(9999)
+          expect {
+            consumer.store_offset(message)
+          }.to raise_error Rdkafka::RdkafkaError
+        end
       end
-      expect {
-        consumer.committed(list)
-      }.to raise_error Rdkafka::RdkafkaError
     end
   end
 
@@ -306,6 +344,18 @@ describe Rdkafka::Consumer do
       }
       expect(lag).to eq(expected_lag)
     end
+
+    it "returns nil if there are no messages on the topic" do
+      list = consumer.committed(Rdkafka::Consumer::TopicPartitionList.new.tap do |l|
+        l.add_topic("consume_test_topic", (0..2))
+      end)
+
+      lag = consumer.lag(list)
+      expected_lag = {
+        "consume_test_topic" => {}
+      }
+      expect(lag).to eq(expected_lag)
+    end
   end
 
   describe "#poll" do
@@ -347,22 +397,23 @@ describe Rdkafka::Consumer do
 
   describe "#each" do
     it "should yield messages" do
+      handles = []
       10.times do
-        producer.produce(
+        handles << producer.produce(
           topic:     "consume_test_topic",
           payload:   "payload 1",
           key:       "key 1",
           partition: 0
-        ).wait
+        )
       end
+      handles.each(&:wait)
 
       consumer.subscribe("consume_test_topic")
-      count = 0
-      # Check the first 10 messages
-      consumer.each do |message|
+      # Check the first 10 messages. Then close the consumer, which
+      # should break the each loop.
+      consumer.each_with_index do |message, i|
         expect(message).to be_a Rdkafka::Consumer::Message
-        count += 1
-        break if count == 10
+        consumer.close if i == 10
       end
     end
   end

data/spec/rdkafka/producer_spec.rb

@@ -8,6 +8,47 @@ describe Rdkafka::Producer do
     expect(Rdkafka::Producer::DeliveryHandle::REGISTRY).to be_empty
   end
 
+  context "delivery callback" do
+    it "should set the callback" do
+      expect {
+        producer.delivery_callback = lambda do |delivery_handle|
+          puts stats
+        end
+      }.not_to raise_error
+      expect(producer.delivery_callback).to be_a Proc
+    end
+
+    it "should not accept a callback that's not a proc" do
+      expect {
+        producer.delivery_callback = 'a string'
+      }.to raise_error(TypeError)
+    end
+
+    it "should call the callback when a message is delivered" do
+      @callback_called = false
+
+      producer.delivery_callback = lambda do |report|
+        expect(report).not_to be_nil
+        expect(report.partition).to eq 1
+        expect(report.offset).to be >= 0
+        @callback_called = true
+      end
+
+      # Produce a message
+      handle = producer.produce(
+        topic:   "produce_test_topic",
+        payload: "payload",
+        key:     "key"
+      )
+
+      # Wait for it to be delivered
+      handle.wait(5)
+
+      # Callback should have been called
+      expect(@callback_called).to be true
+    end
+  end
+
   it "should require a topic" do
     expect {
       producer.produce(

data/spec/spec_helper.rb

@@ -9,6 +9,8 @@ require "rdkafka"
 
 def rdkafka_config(config_overrides={})
   config = {
+    :"api.version.request" => false,
+    :"broker.version.fallback" => "1.0",
     :"bootstrap.servers" => "localhost:9092",
     :"group.id" => "ruby-test-#{Random.new.rand(0..1_000_000)}",
     :"auto.offset.reset" => "earliest",
@@ -36,9 +38,8 @@ def new_native_topic(topic_name="topic_name")
   )
 end
 
-def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, config: nil)
-  config = rdkafka_config if config.nil?
-  consumer = config.consumer
+def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, consumer: nil)
+  consumer = rdkafka_config.consumer if consumer.nil?
   consumer.subscribe(topic)
   timeout = Time.now.to_i + timeout_in_seconds
   loop do
@@ -52,7 +53,11 @@ def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, config: n
       return message
     end
   end
-ensure
-  consumer.commit
-  consumer.close
+end
+
+def wait_for_assignment(consumer)
+  10.times do
+    break if !consumer.assignment.empty?
+    sleep 1
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.4.2
 platform: ruby
 authors:
 - Thijs Cadier
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-19 00:00:00.000000000 Z
+date: 2019-01-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -38,6 +38,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.3'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement