rdkafka 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cd80c3cdbb5bc790349df27bdef345b86e37d341
-  data.tar.gz: 822bad9c78b508e28cc7030443841b622b4d6df2
+  metadata.gz: e665ee94b4d29ec2c4d618540ed4588a6af3cfe8
+  data.tar.gz: 7ad4f0ca3651861441bbe6009a503105e2c7f8c7
 SHA512:
-  metadata.gz: 63f00a3e196e10359220321613b03335bcec067e342155edbe256ef8a04716b435d2444674839e694e423a08a91e240e2e7d0756fb79402a354b8c8ab49cd3d1
-  data.tar.gz: 4e112ca823c2b500187aed86edf540f3eda4fbb45cee3b22ee908a1605696545cff2a77e2b56ba2d73500e185a3c1524992638e6ccb31d8f3c14d32cb783c1bf
+  metadata.gz: 712032c6802290695e4a4be376829c76da6d346b9af8d365e218f3edad511931526553625e7acdf588d1b71779bac2bcbcab43ac0b599ca1be3f592d0aac9759
+  data.tar.gz: 60c30f707e2112aaa63603429208a8fb98778dd5c08d3b29f8c700e6646755fbf1803d1258cf9701ba985636f2eb0eb2eed9bf623dbf174ac85bd2a3fc1bc21a

data/.gitignore

@@ -5,3 +5,4 @@ ext/librdkafka.*
 *.gem
 .yardoc
 doc
+coverage

data/.travis.yml

@@ -2,6 +2,10 @@ language: ruby
 
 sudo: false
 
+env:
+  global:
+    - CC_TEST_REPORTER_ID=0af2487f1cc190d4a5e23cd76182514e5b62aac5d59b6208a02fd518487b9ed8
+
 rvm:
   - 2.1
   - 2.2
@@ -17,6 +21,12 @@ before_install:
 before_script:
   - cd ext && bundle exec rake && cd ..
   - bundle exec rake create_topics
+  - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
+  - chmod +x ./cc-test-reporter
+  - ./cc-test-reporter before-build
 
 script:
   - bundle exec rspec
+
+after_script:
+  - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+# 0.3.1
+* Bump librdkafka to 0.11.1
+* Officially support ranges in `add_topic` for topic partition list.
+* Add consumer lag calculator
+
 # 0.3.0
 * Move both add topic methods to one `add_topic` in `TopicPartitionList`
 * Add committed offsets to consumer

data/README.md

@@ -2,6 +2,8 @@
 
 [![Build Status](https://travis-ci.org/thijsc/rdkafka-ruby.svg?branch=master)](https://travis-ci.org/thijsc/rdkafka-ruby)
 [![Gem Version](https://badge.fury.io/rb/rdkafka.svg)](https://badge.fury.io/rb/rdkafka)
+[![Maintainability](https://api.codeclimate.com/v1/badges/362d93880c1b6b6f48c6/maintainability)](https://codeclimate.com/github/thijsc/rdkafka-ruby/maintainability)
+[![Test Coverage](https://api.codeclimate.com/v1/badges/362d93880c1b6b6f48c6/test_coverage)](https://codeclimate.com/github/thijsc/rdkafka-ruby/test_coverage)
 
 The `rdkafka` gem is a modern Kafka client library for Ruby based on
 [librdkafka](https://github.com/edenhill/librdkafka/).
@@ -54,7 +56,8 @@ end
 
 ## Development
 
-Run `bundle` and `cd ext && bundle exec rake && cd ..`. Then
+For development we expect a local zookeeper and kafka instance to be
+running. Run `bundle` and `cd ext && bundle exec rake && cd ..`. Then
 create the topics as expected in the specs: `bundle exec rake create_topics`.
 
 You can then run `bundle exec rspec` to run the tests. To see rdkafka

data/Rakefile

@@ -8,6 +8,8 @@ task :create_topics do
                    'kafka-topics'
                  end
   `#{kafka_topics} --create --topic=consume_test_topic --zookeeper=127.0.0.1:2181 --partitions=3 --replication-factor=1`
+  `#{kafka_topics} --create --topic=empty_test_topic --zookeeper=127.0.0.1:2181 --partitions=3 --replication-factor=1`
+  `#{kafka_topics} --create --topic=load_test_topic --zookeeper=127.0.0.1:2181 --partitions=3 --replication-factor=1`
   `#{kafka_topics} --create --topic=produce_test_topic --zookeeper=127.0.0.1:2181 --partitions=3 --replication-factor=1`
   `#{kafka_topics} --create --topic=rake_test_topic --zookeeper=127.0.0.1:2181 --partitions=3 --replication-factor=1`
 end
@@ -44,3 +46,49 @@ task :consume_messages do
     puts "Message received: #{message}"
   end
 end
+
+task :load_test do
+  puts "Starting load test"
+
+  config = Rdkafka::Config.new(
+    :"bootstrap.servers" => "localhost:9092",
+    :"group.id" => "load-test",
+    :"enable.partition.eof" => false
+  )
+
+  # Create a producer in a thread
+  Thread.new do
+    producer = config.producer
+    loop do
+      handles = []
+      1000.times do |i|
+        handles.push(producer.produce(
+          topic:   "load_test_topic",
+          payload: "Payload #{i}",
+          key:     "Key #{i}"
+        ))
+      end
+      handles.each(&:wait)
+      puts "Produced 1000 messages"
+    end
+  end.abort_on_exception = true
+
+  # Create three consumers in threads
+  3.times do |i|
+    Thread.new do
+      count = 0
+      consumer = config.consumer
+      consumer.subscribe("load_test_topic")
+      consumer.each do |message|
+        count += 1
+        if count % 1000 == 0
+          puts "Received 1000 messages in thread #{i}"
+        end
+      end
+    end.abort_on_exception = true
+  end
+
+  loop do
+    sleep 1
+  end
+end

data/lib/rdkafka/config.rb

@@ -136,7 +136,7 @@ module Rdkafka
         256
       )
 
-      if handle.nil?
+      if handle.null?
         raise ClientCreationError.new(error_buffer.read_string)
       end
 

data/lib/rdkafka/consumer.rb

@@ -75,6 +75,8 @@ module Rdkafka
     # Return the current committed offset per partition for this consumer group.
     # The offset field of each requested partition will either be set to stored offset or to -1001 in case there was no stored offset for that partition.
     #
+    # TODO: This should use the subscription or assignment by default.
+    #
     # @param list [TopicPartitionList] The topic with partitions to get the offsets for.
     # @param timeout_ms [Integer] The timeout for fetching this information.
     #
@@ -118,7 +120,37 @@ module Rdkafka
         raise Rdkafka::RdkafkaError.new(response)
       end
 
-      return low.read_int, high.read_int
+      return low.read_int64, high.read_int64
+    end
+
+    # Calculate the consumer lag per partition for the provided topic partition list.
+    # You can get a suitable list by calling {committed} or {position} (TODO). It is also
+    # possible to create one yourself, in this case you have to provide a list that
+    # already contains all the partitions you need the lag for.
+    #
+    # @param topic_partition_list [TopicPartitionList] The list to calculate lag for.
+    # @param watermark_timeout_ms [Integer] The timeout for each query watermark call.
+    #
+    # @raise [RdkafkaError] When querying the broker fails.
+    #
+    # @return [Hash<String, Hash<Integer, Integer>>] A hash containing all topics with the lag per partition
+    def lag(topic_partition_list, watermark_timeout_ms=100)
+      out = {}
+      topic_partition_list.to_h.each do |topic, partitions|
+        # Query high watermarks for this topic's partitions
+        # and compare to the offset in the list.
+        topic_out = {}
+        partitions.each do |p|
+          low, high = query_watermark_offsets(
+            topic,
+            p.partition,
+            watermark_timeout_ms
+          )
+          topic_out[p.partition] = high - p.offset
+        end
+        out[topic] = topic_out
+      end
+      out
     end
 
     # Commit the current offsets of this consumer

data/lib/rdkafka/consumer/topic_partition_list.rb

@@ -31,8 +31,14 @@ module Rdkafka
 
       # Add a topic with optionally partitions to the list.
       #
+      # @example Add a topic with unassigned partitions
+      #   tpl.add_topic("topic")
+      #
+      # @example Add a topic with assigned partitions
+      #   tpl.add_topic("topic", (0..8))
+      #
       # @param topic [String] The topic's name
-      # @param partition [Array<Integer>] The topic's partition's
+      # @param partition [Array<Integer>, Range<Integer>] The topic's partition's
       #
       # @return [nil]
       def add_topic(topic, partitions=nil)

data/lib/rdkafka/version.rb

@@ -1,4 +1,4 @@
 module Rdkafka
-  VERSION = "0.3.0"
-  LIBRDKAFKA_VERSION = "0.11.0"
+  VERSION = "0.3.1"
+  LIBRDKAFKA_VERSION = "0.11.1"
 end

data/rdkafka.gemspec

@@ -23,4 +23,5 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency 'pry', '~> 0.10'
   gem.add_development_dependency 'rspec', '~> 3.5'
   gem.add_development_dependency 'rake', '~> 12.0'
+  gem.add_development_dependency 'simplecov', '~> 0.15'
 end

data/spec/rdkafka/bindings_spec.rb

@@ -5,9 +5,58 @@ describe Rdkafka::Bindings do
     expect(Rdkafka::Bindings.ffi_libraries.map(&:name).first).to include "librdkafka"
   end
 
+  describe ".lib_extension" do
+    it "should know the lib extension for darwin" do
+      expect(Gem::Platform.local).to receive(:os).and_return("darwin-aaa")
+      expect(Rdkafka::Bindings.lib_extension).to eq "dylib"
+    end
+
+    it "should know the lib extension for linux" do
+      expect(Gem::Platform.local).to receive(:os).and_return("linux")
+      expect(Rdkafka::Bindings.lib_extension).to eq "so"
+    end
+  end
+
   it "should successfully call librdkafka" do
     expect {
       Rdkafka::Bindings.rd_kafka_conf_new
     }.not_to raise_error
   end
+
+  describe "log callback" do
+    let(:log) { StringIO.new }
+    before do
+      Rdkafka::Config.logger = Logger.new(log)
+    end
+
+    it "should log fatal messages" do
+      Rdkafka::Bindings::LogCallback.call(nil, 0, nil, "log line")
+      expect(log.string).to include "FATAL -- : rdkafka: log line"
+    end
+
+    it "should log error messages" do
+      Rdkafka::Bindings::LogCallback.call(nil, 3, nil, "log line")
+      expect(log.string).to include "ERROR -- : rdkafka: log line"
+    end
+
+    it "should log warning messages" do
+      Rdkafka::Bindings::LogCallback.call(nil, 4, nil, "log line")
+      expect(log.string).to include "WARN -- : rdkafka: log line"
+    end
+
+    it "should log info messages" do
+      Rdkafka::Bindings::LogCallback.call(nil, 5, nil, "log line")
+      expect(log.string).to include "INFO -- : rdkafka: log line"
+    end
+
+    it "should log debug messages" do
+      Rdkafka::Bindings::LogCallback.call(nil, 7, nil, "log line")
+      expect(log.string).to include "DEBUG -- : rdkafka: log line"
+    end
+
+    it "should log unknown messages" do
+      Rdkafka::Bindings::LogCallback.call(nil, 100, nil, "log line")
+      expect(log.string).to include "ANY -- : rdkafka: log line"
+    end
+  end
 end

data/spec/rdkafka/config_spec.rb

@@ -53,5 +53,25 @@ describe Rdkafka::Config do
         config.producer
       }.to raise_error(Rdkafka::Config::ConfigError, "No such configuration property: \"invalid.key\"")
     end
+
+    it "should raise an error when client creation fails for a consumer" do
+      config = Rdkafka::Config.new(
+        "security.protocol" => "SSL",
+        "ssl.ca.location" => "/nonsense"
+      )
+      expect {
+        config.consumer
+      }.to raise_error(Rdkafka::Config::ClientCreationError, /ssl.ca.location failed(.*)/)
+    end
+
+    it "should raise an error when client creation fails for a producer" do
+      config = Rdkafka::Config.new(
+        "security.protocol" => "SSL",
+        "ssl.ca.location" => "/nonsense"
+      )
+      expect {
+        config.producer
+      }.to raise_error(Rdkafka::Config::ClientCreationError, /ssl.ca.location failed(.*)/)
+    end
   end
 end

data/spec/rdkafka/consumer/message_spec.rb

@@ -1,13 +1,7 @@
 require "spec_helper"
 
 describe Rdkafka::Consumer::Message do
-  let(:native_topic) do
-    Rdkafka::Bindings.rd_kafka_topic_new(
-      native_client,
-      "topic_name",
-      nil
-    )
-  end
+  let(:native_topic) { new_native_topic }
   let(:payload) { nil }
   let(:key) { nil }
   let(:native_message) do

data/spec/rdkafka/consumer/topic_partition_list_spec.rb

@@ -37,7 +37,32 @@ describe Rdkafka::Consumer::TopicPartitionList do
     })
   end
 
-  it "should create a new list and add assigned topics" do
+  it "should create a new list and add assigned topics as a range" do
+    list = Rdkafka::Consumer::TopicPartitionList.new
+
+    expect(list.count).to eq 0
+    expect(list.empty?).to be true
+
+    list.add_topic("topic1", (0..2))
+    list.add_topic("topic2", (0..1))
+
+    expect(list.count).to eq 5
+    expect(list.empty?).to be false
+
+    hash = list.to_h
+    expect(hash.count).to eq 2
+    expect(hash["topic1"]).to eq([
+      Rdkafka::Consumer::Partition.new(0, -1001),
+      Rdkafka::Consumer::Partition.new(1, -1001),
+      Rdkafka::Consumer::Partition.new(2, -1001)
+    ])
+    expect(hash["topic2"]).to eq([
+      Rdkafka::Consumer::Partition.new(0, -1001),
+      Rdkafka::Consumer::Partition.new(1, -1001)
+    ])
+  end
+
+  it "should create a new list and add assigned topics as an array" do
     list = Rdkafka::Consumer::TopicPartitionList.new
 
     expect(list.count).to eq 0

data/spec/rdkafka/consumer_spec.rb

@@ -5,8 +5,8 @@ describe Rdkafka::Consumer do
   let(:consumer) { config.consumer }
   let(:producer) { config.producer }
 
-  context "subscription" do
-    it "should subscribe" do
+  describe "#subscripe, #unsubscribe and #subscription" do
+    it "should subscribe, unsubscribe and return the subscription" do
       expect(consumer.subscription).to be_empty
 
       consumer.subscribe("consume_test_topic")
@@ -21,9 +21,41 @@ describe Rdkafka::Consumer do
 
       expect(consumer.subscription).to be_empty
     end
+
+    it "should raise an error when subscribing fails" do
+      expect(Rdkafka::Bindings).to receive(:rd_kafka_subscribe).and_return(20)
+
+      expect {
+        consumer.subscribe("consume_test_topic")
+      }.to raise_error(Rdkafka::RdkafkaError)
+    end
+
+    it "should raise an error when unsubscribing fails" do
+      expect(Rdkafka::Bindings).to receive(:rd_kafka_unsubscribe).and_return(20)
+
+      expect {
+        consumer.unsubscribe
+      }.to raise_error(Rdkafka::RdkafkaError)
+    end
+
+    it "should raise an error when fetching the subscription fails" do
+      expect(Rdkafka::Bindings).to receive(:rd_kafka_subscription).and_return(20)
+
+      expect {
+        consumer.subscription
+      }.to raise_error(Rdkafka::RdkafkaError)
+    end
+  end
+
+  context "#close" do
+    it "should close a consumer" do
+      consumer.subscribe("consume_test_topic")
+      consumer.close
+      expect(consumer.poll(100)).to be_nil
+    end
   end
 
-  describe "committed" do
+  describe "#commit and #committed" do
     before do
       # Make sure there's a stored offset
       report = producer.produce(
@@ -32,6 +64,8 @@ describe Rdkafka::Consumer do
         key:       "key 1",
         partition: 0
       ).wait
+      # Wait for message commits the current state,
+      # commit is therefore tested here.
       message = wait_for_message(
         topic: "consume_test_topic",
         delivery_report: report,
@@ -39,6 +73,20 @@ describe Rdkafka::Consumer do
       )
     end
 
+    it "should only accept a topic partition list" do
+      expect {
+        consumer.committed("list")
+      }.to raise_error TypeError
+    end
+
+    it "should raise an error when committing fails" do
+      expect(Rdkafka::Bindings).to receive(:rd_kafka_commit).and_return(20)
+
+      expect {
+        consumer.commit
+      }.to raise_error(Rdkafka::RdkafkaError)
+    end
+
     it "should fetch the committed offsets for a specified topic partition list" do
       list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
         list.add_topic("consume_test_topic", [0, 1, 2])
@@ -48,9 +96,19 @@ describe Rdkafka::Consumer do
       expect(partitions[1].offset).to eq -1001
       expect(partitions[2].offset).to eq -1001
     end
+
+    it "should raise an error when getting committed fails" do
+      expect(Rdkafka::Bindings).to receive(:rd_kafka_committed).and_return(20)
+      list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+        list.add_topic("consume_test_topic", [0, 1, 2])
+      end
+      expect {
+        consumer.committed(list)
+      }.to raise_error Rdkafka::RdkafkaError
+    end
   end
 
-  describe "watermark offsets" do
+  describe "#query_watermark_offsets" do
     it "should return the watermark offsets" do
       # Make sure there's a message
       producer.produce(
@@ -64,5 +122,141 @@ describe Rdkafka::Consumer do
       expect(low).to eq 0
       expect(high).to be > 0
     end
+
+    it "should raise an error when querying offsets fails" do
+      expect(Rdkafka::Bindings).to receive(:rd_kafka_query_watermark_offsets).and_return(20)
+      expect {
+        consumer.query_watermark_offsets("consume_test_topic", 0, 5000)
+      }.to raise_error Rdkafka::RdkafkaError
+    end
+  end
+
+  describe "#lag" do
+    let(:config) { rdkafka_config(:"enable.partition.eof" => true) }
+
+    it "should calculate the consumer lag" do
+      # Make sure there's a message in every partition and
+      # wait for the message to make sure everything is committed.
+      (0..2).each do |i|
+        report = producer.produce(
+          topic:     "consume_test_topic",
+          key:       "key lag #{i}",
+          partition: i
+        ).wait
+      end
+
+      # Consume to the end
+      consumer.subscribe("consume_test_topic")
+      eof_count = 0
+      loop do
+        begin
+          consumer.poll(100)
+        rescue Rdkafka::RdkafkaError => error
+          if error.is_partition_eof?
+            eof_count += 1
+          end
+          break if eof_count == 3
+        end
+      end
+
+      # Commit
+      consumer.commit
+
+      # Create list to fetch lag for. TODO creating the list will not be necessary
+      # after committed uses the subscription.
+      list = consumer.committed(Rdkafka::Consumer::TopicPartitionList.new.tap do |l|
+        l.add_topic("consume_test_topic", (0..2))
+      end)
+
+      # Lag should be 0 now
+      lag = consumer.lag(list)
+      expected_lag = {
+        "consume_test_topic" => {
+          0 => 0,
+          1 => 0,
+          2 => 0
+        }
+      }
+      expect(lag).to eq(expected_lag)
+
+      # Produce message on every topic again
+      (0..2).each do |i|
+        report = producer.produce(
+          topic:     "consume_test_topic",
+          key:       "key lag #{i}",
+          partition: i
+        ).wait
+      end
+
+      # Lag should be 1 now
+      lag = consumer.lag(list)
+      expected_lag = {
+        "consume_test_topic" => {
+          0 => 1,
+          1 => 1,
+          2 => 1
+        }
+      }
+      expect(lag).to eq(expected_lag)
+    end
+  end
+
+  describe "#poll" do
+    it "should return nil if there is no subscription" do
+      expect(consumer.poll(1000)).to be_nil
+    end
+
+    it "should return nil if there are no messages" do
+      consumer.subscribe("empty_test_topic")
+      expect(consumer.poll(1000)).to be_nil
+    end
+
+    it "should return a message if there is one" do
+      producer.produce(
+        topic:     "consume_test_topic",
+        payload:   "payload 1",
+        key:       "key 1"
+      ).wait
+
+      consumer.subscribe("consume_test_topic")
+      message = consumer.poll(5000)
+      expect(message).to be_a Rdkafka::Consumer::Message
+
+      # Message content is tested in producer spec
+    end
+
+    it "should raise an error when polling fails" do
+      message = Rdkafka::Bindings::Message.new.tap do |message|
+        message[:err] = 20
+      end
+      message_pointer = message.to_ptr
+      expect(Rdkafka::Bindings).to receive(:rd_kafka_consumer_poll).and_return(message_pointer)
+      expect(Rdkafka::Bindings).to receive(:rd_kafka_message_destroy).with(message_pointer)
+      expect {
+        consumer.poll(100)
+      }.to raise_error Rdkafka::RdkafkaError
+    end
+  end
+
+  describe "#each" do
+    it "should yield messages" do
+      10.times do
+        producer.produce(
+          topic:     "consume_test_topic",
+          payload:   "payload 1",
+          key:       "key 1",
+          partition: 0
+        ).wait
+      end
+
+      consumer.subscribe("consume_test_topic")
+      count = 0
+      # Check the first 10 messages
+      consumer.each do |message|
+        expect(message).to be_a Rdkafka::Consumer::Message
+        count += 1
+        break if count == 10
+      end
+    end
   end
 end

data/spec/rdkafka/producer/delivery_handle_spec.rb

@@ -47,6 +47,13 @@ describe Rdkafka::Producer::DeliveryHandle do
         expect(report.partition).to eq(2)
         expect(report.offset).to eq(100)
       end
+
+      it "should wait without a timeout" do
+        report = subject.wait(nil)
+
+        expect(report.partition).to eq(2)
+        expect(report.offset).to eq(100)
+      end
     end
 
     context "when not pending anymore and there was an error" do

data/spec/rdkafka/producer_spec.rb

@@ -13,19 +13,22 @@ describe Rdkafka::Producer do
   end
 
   it "should produce a message" do
+    # Produce a message
     handle = producer.produce(
       topic:   "produce_test_topic",
-      payload: "payload 1",
-      key:     "key 1"
+      payload: "payload",
+      key:     "key"
     )
+
+    # Should be pending at first
     expect(handle.pending?).to be true
 
     # Check delivery handle and report
     report = handle.wait(5)
     expect(handle.pending?).to be false
     expect(report).not_to be_nil
-    expect(report.partition).to eq 0
-    expect(report.offset).to be > 0
+    expect(report.partition).to eq 1
+    expect(report.offset).to be >= 0
 
     # Close producer
     producer.close
@@ -35,28 +38,41 @@ describe Rdkafka::Producer do
       topic: "produce_test_topic",
       delivery_report: report
     )
-    expect(message.partition).to eq 0
-    expect(message.payload).to eq "payload 1"
-    expect(message.key).to eq "key 1"
+    expect(message.partition).to eq 1
+    expect(message.payload).to eq "payload"
+    expect(message.key).to eq "key"
     # Since api.version.request is on by default we will get
     # the message creation timestamp if it's not set.
     expect(message.timestamp).to be > 1505069891557
   end
 
+  it "should produce a message with a specified partition" do
+    # Produce a message
+    handle = producer.produce(
+      topic:     "produce_test_topic",
+      payload:   "payload partition",
+      key:       "key partition",
+      partition: 1
+    )
+    report = handle.wait(5)
+
+    # Consume message and verify it's content
+    message = wait_for_message(
+      topic: "produce_test_topic",
+      delivery_report: report
+    )
+    expect(message.partition).to eq 1
+    expect(message.key).to eq "key partition"
+  end
+
   it "should produce a message with utf-8 encoding" do
     handle = producer.produce(
       topic:   "produce_test_topic",
       payload: "Τη γλώσσα μου έδωσαν ελληνική",
       key:     "key utf8"
     )
-    expect(handle.pending?).to be true
-
-    # Check delivery handle and report
     report = handle.wait(5)
 
-    # Close producer
-    producer.close
-
     # Consume message and verify it's content
     message = wait_for_message(
       topic: "produce_test_topic",
@@ -71,18 +87,12 @@ describe Rdkafka::Producer do
   it "should produce a message with a timestamp" do
     handle = producer.produce(
       topic:     "produce_test_topic",
-      payload:   "Payload timestamp",
+      payload:   "payload timestamp",
       key:       "key timestamp",
       timestamp: 1505069646000
     )
-    expect(handle.pending?).to be true
-
-    # Check delivery handle and report
     report = handle.wait(5)
 
-    # Close producer
-    producer.close
-
     # Consume message and verify it's content
     message = wait_for_message(
       topic: "produce_test_topic",
@@ -94,11 +104,56 @@ describe Rdkafka::Producer do
     expect(message.timestamp).to eq 1505069646000
   end
 
+  it "should produce a message with nil key" do
+    handle = producer.produce(
+      topic:   "produce_test_topic",
+      payload: "payload no key"
+    )
+    report = handle.wait(5)
+
+    # Consume message and verify it's content
+    message = wait_for_message(
+      topic: "produce_test_topic",
+      delivery_report: report
+    )
+
+    expect(message.key).to be_nil
+    expect(message.payload).to eq "payload no key"
+  end
+
+  it "should produce a message with nil payload" do
+    handle = producer.produce(
+      topic: "produce_test_topic",
+      key:   "key no payload"
+    )
+    report = handle.wait(5)
+
+    # Consume message and verify it's content
+    message = wait_for_message(
+      topic: "produce_test_topic",
+      delivery_report: report
+    )
+
+    expect(message.key).to eq "key no payload"
+    expect(message.payload).to be_nil
+  end
+
+  it "should raise an error when producing fails" do
+    expect(Rdkafka::Bindings).to receive(:rd_kafka_producev).and_return(20)
+
+    expect {
+      producer.produce(
+        topic:   "produce_test_topic",
+        key:     "key error"
+      )
+    }.to raise_error Rdkafka::RdkafkaError
+  end
+
   it "should raise a timeout error when waiting too long" do
     handle = producer.produce(
       topic:   "produce_test_topic",
-      payload: "payload 1",
-      key:     "key 1"
+      payload: "payload timeout",
+      key:     "key timeout"
     )
     expect {
       handle.wait(0)

data/spec/spec_helper.rb

@@ -1,8 +1,13 @@
+require "simplecov"
+SimpleCov.start do
+  add_filter "/spec/"
+end
+
 require "pry"
 require "rspec"
 require "rdkafka"
 
-def rdkafka_config
+def rdkafka_config(config_overrides={})
   config = {
     :"bootstrap.servers" => "localhost:9092",
     :"group.id" => "ruby-test-#{Random.new.rand(0..1_000_000)}",
@@ -14,6 +19,7 @@ def rdkafka_config
   elsif ENV["DEBUG_CONSUMER"]
     config[:debug] = "cgrp,topic,fetch"
   end
+  config.merge!(config_overrides)
   Rdkafka::Config.new(config)
 end
 
@@ -22,6 +28,14 @@ def native_client
   config.send(:native_kafka, config.send(:native_config), :rd_kafka_producer)
 end
 
+def new_native_topic(topic_name="topic_name")
+  Rdkafka::Bindings.rd_kafka_topic_new(
+    native_client,
+    topic_name,
+    nil
+  )
+end
+
 def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, config: nil)
   config = rdkafka_config if config.nil?
   consumer = config.consumer
@@ -34,7 +48,7 @@ def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, config: n
     message = consumer.poll(100)
     if message &&
         message.partition == delivery_report.partition &&
-        message.offset == delivery_report.offset - 1
+        message.offset == delivery_report.offset
       return message
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Thijs Cadier
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-17 00:00:00.000000000 Z
+date: 2017-10-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -80,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '12.0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.15'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.15'
 description: Modern Kafka client library for Ruby based on librdkafka
 email:
 - thijs@appsignal.com