rdkafka 0.5.0 → 0.8.1

data/spec/rdkafka/bindings_spec.rb

@@ -1,4 +1,5 @@
 require "spec_helper"
+require 'zlib'
 
 describe Rdkafka::Bindings do
   it "should load librdkafka" do
@@ -7,12 +8,12 @@ describe Rdkafka::Bindings do
 
   describe ".lib_extension" do
     it "should know the lib extension for darwin" do
-      expect(Gem::Platform.local).to receive(:os).and_return("darwin-aaa")
+      stub_const('RbConfig::CONFIG', 'host_os' =>'darwin')
       expect(Rdkafka::Bindings.lib_extension).to eq "dylib"
     end
 
     it "should know the lib extension for linux" do
-      expect(Gem::Platform.local).to receive(:os).and_return("linux")
+      stub_const('RbConfig::CONFIG', 'host_os' =>'linux')
       expect(Rdkafka::Bindings.lib_extension).to eq "so"
     end
   end
@@ -60,6 +61,23 @@ describe Rdkafka::Bindings do
     end
   end
 
+  describe "partitioner" do
+    let(:partition_key) { ('a'..'z').to_a.shuffle.take(15).join('') }
+    let(:partition_count) { rand(50) + 1 }
+
+    it "should return the same partition for a similar string and the same partition count" do
+      result_1 = Rdkafka::Bindings.partitioner(partition_key, partition_count)
+      result_2 = Rdkafka::Bindings.partitioner(partition_key, partition_count)
+      expect(result_1).to eq(result_2)
+    end
+
+    it "should match the old partitioner" do
+      result_1 = Rdkafka::Bindings.partitioner(partition_key, partition_count)
+      result_2 = (Zlib.crc32(partition_key) % partition_count)
+      expect(result_1).to eq(result_2)
+    end
+  end
+
   describe "stats callback" do
     context "without a stats callback" do
       it "should do nothing" do

data/spec/rdkafka/callbacks_spec.rb

@@ -0,0 +1,20 @@
+require "spec_helper"
+
+describe Rdkafka::Callbacks do
+
+  # The code in the call back functions is 100% covered by other specs.  Due to
+  # the large number of collaborators, and the fact that FFI does not play
+  # nicely with doubles, it was very difficult to construct tests that were
+  # not over-mocked.
+
+  # For debugging purposes, if you suspect that you are running into trouble in
+  # one of the callback functions, it may be helpful to surround the inner body
+  # of the method with something like:
+  #
+  #   begin
+  #     <method body>
+  #   rescue => ex; puts ex.inspect; puts ex.backtrace; end;
+  #
+  # This will output to STDOUT any exceptions that are being raised in the callback.
+
+end

data/spec/rdkafka/config_spec.rb

@@ -50,7 +50,9 @@ describe Rdkafka::Config do
     end
 
     it "should create a consumer with valid config" do
-      expect(rdkafka_config.consumer).to be_a Rdkafka::Consumer
+      consumer = rdkafka_config.consumer
+      expect(consumer).to be_a Rdkafka::Consumer
+      consumer.close
     end
 
     it "should raise an error when creating a consumer with invalid config" do
@@ -76,7 +78,9 @@ describe Rdkafka::Config do
     end
 
     it "should create a producer with valid config" do
-      expect(rdkafka_config.producer).to be_a Rdkafka::Producer
+      producer = rdkafka_config.producer
+      expect(producer).to be_a Rdkafka::Producer
+      producer.close
     end
 
     it "should raise an error when creating a producer with invalid config" do
@@ -86,6 +90,17 @@ describe Rdkafka::Config do
       }.to raise_error(Rdkafka::Config::ConfigError, "No such configuration property: \"invalid.key\"")
     end
 
+    it "should allow configuring zstd compression" do
+      config = Rdkafka::Config.new('compression.codec' => 'zstd')
+      begin
+        expect(config.producer).to be_a Rdkafka::Producer
+        config.producer.close
+      rescue Rdkafka::Config::ConfigError => ex
+        pending "Zstd compression not supported on this machine"
+        raise ex
+      end
+    end
+
     it "should raise an error when client creation fails for a consumer" do
       config = Rdkafka::Config.new(
         "security.protocol" => "SSL",

data/spec/rdkafka/consumer/message_spec.rb

@@ -1,7 +1,8 @@
 require "spec_helper"
 
 describe Rdkafka::Consumer::Message do
-  let(:native_topic) { new_native_topic }
+  let(:native_client) { new_native_client }
+  let(:native_topic) { new_native_topic(native_client: native_client) }
   let(:payload) { nil }
   let(:key) { nil }
   let(:native_message) do
@@ -24,6 +25,10 @@ describe Rdkafka::Consumer::Message do
     end
   end
 
+  after(:each) do
+    Rdkafka::Bindings.rd_kafka_destroy(native_client)
+  end
+
   subject { Rdkafka::Consumer::Message.new(native_message) }
 
   before do

data/spec/rdkafka/consumer_spec.rb

@@ -1,11 +1,15 @@
 require "spec_helper"
+require "ostruct"
 
 describe Rdkafka::Consumer do
   let(:config) { rdkafka_config }
   let(:consumer) { config.consumer }
   let(:producer) { config.producer }
 
-  describe "#subscripe, #unsubscribe and #subscription" do
+  after { consumer.close }
+  after { producer.close }
+
+  describe "#subscribe, #unsubscribe and #subscription" do
     it "should subscribe, unsubscribe and return the subscription" do
       expect(consumer.subscription).to be_empty
 
@@ -75,7 +79,7 @@ describe Rdkafka::Consumer do
         tpl.add_topic("consume_test_topic", (0..2))
         consumer.pause(tpl)
 
-        # 6. unsure that messages are not available
+        # 6. ensure that messages are not available
         records = consumer.poll(timeout)
         expect(records).to be_nil
 
@@ -84,10 +88,9 @@ describe Rdkafka::Consumer do
         tpl.add_topic("consume_test_topic", (0..2))
         consumer.resume(tpl)
 
-        # 8. ensure that message is successfuly consumed
+        # 8. ensure that message is successfully consumed
         records = consumer.poll(timeout)
         expect(records).not_to be_nil
-        consumer.commit
       end
     end
 
@@ -124,6 +127,90 @@ describe Rdkafka::Consumer do
     end
   end
 
+  describe "#seek" do
+    it "should raise an error when seeking fails" do
+      fake_msg = OpenStruct.new(topic: "consume_test_topic", partition: 0, offset: 0)
+
+      expect(Rdkafka::Bindings).to receive(:rd_kafka_seek).and_return(20)
+      expect {
+        consumer.seek(fake_msg)
+      }.to raise_error Rdkafka::RdkafkaError
+    end
+
+    context "subscription" do
+      let(:timeout) { 1000 }
+
+      before do
+        consumer.subscribe("consume_test_topic")
+
+        # 1. partitions are assigned
+        wait_for_assignment(consumer)
+        expect(consumer.assignment).not_to be_empty
+
+        # 2. eat unrelated messages
+        while(consumer.poll(timeout)) do; end
+      end
+      after { consumer.unsubscribe }
+
+      def send_one_message(val)
+        producer.produce(
+          topic:     "consume_test_topic",
+          payload:   "payload #{val}",
+          key:       "key 1",
+          partition: 0
+        ).wait
+      end
+
+      it "works when a partition is paused" do
+        # 3. get reference message
+        send_one_message(:a)
+        message1 = consumer.poll(timeout)
+        expect(message1&.payload).to eq "payload a"
+
+        # 4. pause the subscription
+        tpl = Rdkafka::Consumer::TopicPartitionList.new
+        tpl.add_topic("consume_test_topic", 1)
+        consumer.pause(tpl)
+
+        # 5. seek to previous message
+        consumer.seek(message1)
+
+        # 6. resume the subscription
+        tpl = Rdkafka::Consumer::TopicPartitionList.new
+        tpl.add_topic("consume_test_topic", 1)
+        consumer.resume(tpl)
+
+        # 7. ensure same message is read again
+        message2 = consumer.poll(timeout)
+        consumer.commit
+        expect(message1.offset).to eq message2.offset
+        expect(message1.payload).to eq message2.payload
+      end
+
+      it "allows skipping messages" do
+        # 3. send messages
+        send_one_message(:a)
+        send_one_message(:b)
+        send_one_message(:c)
+
+        # 4. get reference message
+        message = consumer.poll(timeout)
+        expect(message&.payload).to eq "payload a"
+
+        # 5. seek over one message
+        fake_msg = message.dup
+        fake_msg.instance_variable_set(:@offset, fake_msg.offset + 2)
+        consumer.seek(fake_msg)
+
+        # 6. ensure that only one message is available
+        records = consumer.poll(timeout)
+        expect(records&.payload).to eq "payload c"
+        records = consumer.poll(timeout)
+        expect(records).to be_nil
+      end
+    end
+  end
+
   describe "#assign and #assignment" do
     it "should return an empty assignment if nothing is assigned" do
       expect(consumer.assignment).to be_empty
@@ -185,7 +272,9 @@ describe Rdkafka::Consumer do
     it "should close a consumer" do
       consumer.subscribe("consume_test_topic")
       consumer.close
-      expect(consumer.poll(100)).to be_nil
+      expect {
+        consumer.poll(100)
+      }.to raise_error(Rdkafka::ClosedConsumerError, /poll/)
     end
   end
 
@@ -226,11 +315,11 @@ describe Rdkafka::Consumer do
       }.to raise_error TypeError
     end
 
-    context "with a commited consumer" do
+    context "with a committed consumer" do
       before :all do
-        # Make sure there are some message
-        producer = rdkafka_config.producer
+        # Make sure there are some messages.
         handles = []
+        producer = rdkafka_config.producer
         10.times do
           (0..2).each do |i|
             handles << producer.produce(
@@ -242,6 +331,7 @@ describe Rdkafka::Consumer do
           end
         end
         handles.each(&:wait)
+        producer.close
       end
 
       before do
@@ -302,20 +392,26 @@ describe Rdkafka::Consumer do
 
       describe "#store_offset" do
         before do
+          config = {}
           config[:'enable.auto.offset.store'] = false
           config[:'enable.auto.commit'] = false
-          consumer.subscribe("consume_test_topic")
-          wait_for_assignment(consumer)
+          @new_consumer = rdkafka_config(config).consumer
+          @new_consumer.subscribe("consume_test_topic")
+          wait_for_assignment(@new_consumer)
+        end
+
+        after do
+          @new_consumer.close
         end
 
         it "should store the offset for a message" do
-          consumer.store_offset(message)
-          consumer.commit
+          @new_consumer.store_offset(message)
+          @new_consumer.commit
 
           list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
             list.add_topic("consume_test_topic", [0, 1, 2])
           end
-          partitions = consumer.committed(list).to_h["consume_test_topic"]
+          partitions = @new_consumer.committed(list).to_h["consume_test_topic"]
           expect(partitions).not_to be_nil
           expect(partitions[message.partition].offset).to eq(message.offset + 1)
         end
@@ -323,7 +419,7 @@ describe Rdkafka::Consumer do
         it "should raise an error with invalid input" do
           allow(message).to receive(:partition).and_return(9999)
           expect {
-            consumer.store_offset(message)
+            @new_consumer.store_offset(message)
           }.to raise_error Rdkafka::RdkafkaError
         end
       end
@@ -467,12 +563,12 @@ describe Rdkafka::Consumer do
         payload:   "payload 1",
         key:       "key 1"
       ).wait
-
       consumer.subscribe("consume_test_topic")
-      message = consumer.poll(5000)
-      expect(message).to be_a Rdkafka::Consumer::Message
+      message = consumer.each {|m| break m}
 
-      # Message content is tested in producer spec
+      expect(message).to be_a Rdkafka::Consumer::Message
+      expect(message.payload).to eq('payload 1')
+      expect(message.key).to eq('key 1')
     end
 
     it "should raise an error when polling fails" do
@@ -568,8 +664,9 @@ describe Rdkafka::Consumer do
       # should break the each loop.
       consumer.each_with_index do |message, i|
         expect(message).to be_a Rdkafka::Consumer::Message
-        consumer.close if i == 10
+        break if i == 10
       end
+      consumer.close
     end
   end
 
@@ -629,4 +726,33 @@ describe Rdkafka::Consumer do
       consumer.close
     end
   end
+
+  context "methods that should not be called after a consumer has been closed" do
+    before do
+      consumer.close
+    end
+
+    # Affected methods and a non-invalid set of parameters for the method
+    {
+        :subscribe               => [ nil ],
+        :unsubscribe             => nil,
+        :pause                   => [ nil ],
+        :resume                  => [ nil ],
+        :subscription            => nil,
+        :assign                  => [ nil ],
+        :assignment              => nil,
+        :committed               => [],
+        :query_watermark_offsets => [ nil, nil ],
+    }.each do |method, args|
+      it "raises an exception if #{method} is called" do
+        expect {
+          if args.nil?
+            consumer.public_send(method)
+          else
+            consumer.public_send(method, *args)
+          end
+        }.to raise_exception(Rdkafka::ClosedConsumerError, /#{method.to_s}/)
+      end
+    end
+  end
 end

data/spec/rdkafka/error_spec.rb

@@ -11,6 +11,10 @@ describe Rdkafka::RdkafkaError do
     expect(Rdkafka::RdkafkaError.new(10, "message prefix").message_prefix).to eq "message prefix"
   end
 
+  it "should create an error with a broker message" do
+    expect(Rdkafka::RdkafkaError.new(10, broker_message: "broker message").broker_message).to eq "broker message"
+  end
+
   describe "#code" do
     it "should handle an invalid response" do
       expect(Rdkafka::RdkafkaError.new(933975).code).to eq :err_933975?
@@ -71,15 +75,15 @@ describe Rdkafka::RdkafkaError do
     end
 
     it "should not equal another error with a different error code" do
-      expect(subject).to eq Rdkafka::RdkafkaError.new(20, "Error explanation")
+      expect(subject).not_to eq Rdkafka::RdkafkaError.new(20, "Error explanation")
     end
 
     it "should not equal another error with a different message" do
-      expect(subject).to eq Rdkafka::RdkafkaError.new(10, "Different error explanation")
+      expect(subject).not_to eq Rdkafka::RdkafkaError.new(10, "Different error explanation")
     end
 
     it "should not equal another error with no message" do
-      expect(subject).to eq Rdkafka::RdkafkaError.new(10)
+      expect(subject).not_to eq Rdkafka::RdkafkaError.new(10)
     end
   end
 end

data/spec/rdkafka/metadata_spec.rb

@@ -0,0 +1,78 @@
+require "spec_helper"
+require "securerandom"
+
+describe Rdkafka::Metadata do
+  let(:config)        { rdkafka_config }
+  let(:native_config) { config.send(:native_config) }
+  let(:native_kafka)  { config.send(:native_kafka, native_config, :rd_kafka_consumer) }
+
+  after do
+    Rdkafka::Bindings.rd_kafka_consumer_close(native_kafka)
+    Rdkafka::Bindings.rd_kafka_destroy(native_kafka)
+  end
+
+  context "passing in a topic name" do
+    context "that is non-existent topic" do
+      let(:topic_name) { SecureRandom.uuid.to_s }
+
+      it "raises an appropriate exception" do
+        expect {
+          described_class.new(native_kafka, topic_name)
+        }.to raise_exception(Rdkafka::RdkafkaError, "Broker: Leader not available (leader_not_available)")
+      end
+    end
+
+    context "that is one of our test topics" do
+      subject { described_class.new(native_kafka, topic_name) }
+      let(:topic_name) { "partitioner_test_topic" }
+
+      it "#brokers returns our single broker" do
+        expect(subject.brokers.length).to eq(1)
+        expect(subject.brokers[0][:broker_id]).to eq(1)
+        expect(subject.brokers[0][:broker_name]).to eq("localhost")
+        expect(subject.brokers[0][:broker_port]).to eq(9092)
+      end
+
+      it "#topics returns data on our test topic" do
+        expect(subject.topics.length).to eq(1)
+        expect(subject.topics[0][:partition_count]).to eq(25)
+        expect(subject.topics[0][:partitions].length).to eq(25)
+        expect(subject.topics[0][:topic_name]).to eq(topic_name)
+      end
+    end
+  end
+
+  context "not passing in a topic name" do
+    subject { described_class.new(native_kafka, topic_name) }
+    let(:topic_name) { nil }
+    let(:test_topics) {
+      %w(consume_test_topic empty_test_topic load_test_topic produce_test_topic rake_test_topic watermarks_test_topic partitioner_test_topic)
+    } # Test topics crated in spec_helper.rb
+
+    it "#brokers returns our single broker" do
+      expect(subject.brokers.length).to eq(1)
+      expect(subject.brokers[0][:broker_id]).to eq(1)
+      expect(subject.brokers[0][:broker_name]).to eq("localhost")
+      expect(subject.brokers[0][:broker_port]).to eq(9092)
+    end
+
+    it "#topics returns data about all of our test topics" do
+      result = subject.topics.map { |topic| topic[:topic_name] }
+      expect(result).to include(*test_topics)
+    end
+  end
+
+  context "when a non-zero error code is returned" do
+    let(:topic_name) { SecureRandom.uuid.to_s }
+
+    before do
+      allow(Rdkafka::Bindings).to receive(:rd_kafka_metadata).and_return(-165)
+    end
+
+    it "creating the instance raises an exception" do
+      expect {
+        described_class.new(native_kafka, topic_name)
+      }.to raise_error(Rdkafka::RdkafkaError, /Local: Required feature not supported by broker \(unsupported_feature\)/)
+    end
+  end
+end