rdkafka 0.6.0 → 0.9.0

data/spec/rdkafka/error_spec.rb

@@ -11,6 +11,10 @@ describe Rdkafka::RdkafkaError do
     expect(Rdkafka::RdkafkaError.new(10, "message prefix").message_prefix).to eq "message prefix"
   end
 
+  it "should create an error with a broker message" do
+    expect(Rdkafka::RdkafkaError.new(10, broker_message: "broker message").broker_message).to eq "broker message"
+  end
+
   describe "#code" do
     it "should handle an invalid response" do
       expect(Rdkafka::RdkafkaError.new(933975).code).to eq :err_933975?
@@ -71,15 +75,15 @@ describe Rdkafka::RdkafkaError do
     end
 
     it "should not equal another error with a different error code" do
-      expect(subject).to eq Rdkafka::RdkafkaError.new(20, "Error explanation")
+      expect(subject).not_to eq Rdkafka::RdkafkaError.new(20, "Error explanation")
     end
 
     it "should not equal another error with a different message" do
-      expect(subject).to eq Rdkafka::RdkafkaError.new(10, "Different error explanation")
+      expect(subject).not_to eq Rdkafka::RdkafkaError.new(10, "Different error explanation")
     end
 
     it "should not equal another error with no message" do
-      expect(subject).to eq Rdkafka::RdkafkaError.new(10)
+      expect(subject).not_to eq Rdkafka::RdkafkaError.new(10)
     end
   end
 end

data/spec/rdkafka/metadata_spec.rb

@@ -0,0 +1,78 @@
+require "spec_helper"
+require "securerandom"
+
+describe Rdkafka::Metadata do
+  let(:config)        { rdkafka_config }
+  let(:native_config) { config.send(:native_config) }
+  let(:native_kafka)  { config.send(:native_kafka, native_config, :rd_kafka_consumer) }
+
+  after do
+    Rdkafka::Bindings.rd_kafka_consumer_close(native_kafka)
+    Rdkafka::Bindings.rd_kafka_destroy(native_kafka)
+  end
+
+  context "passing in a topic name" do
+    context "that is non-existent topic" do
+      let(:topic_name) { SecureRandom.uuid.to_s }
+
+      it "raises an appropriate exception" do
+        expect {
+          described_class.new(native_kafka, topic_name)
+        }.to raise_exception(Rdkafka::RdkafkaError, "Broker: Leader not available (leader_not_available)")
+      end
+    end
+
+    context "that is one of our test topics" do
+      subject { described_class.new(native_kafka, topic_name) }
+      let(:topic_name) { "partitioner_test_topic" }
+
+      it "#brokers returns our single broker" do
+        expect(subject.brokers.length).to eq(1)
+        expect(subject.brokers[0][:broker_id]).to eq(1)
+        expect(subject.brokers[0][:broker_name]).to eq("localhost")
+        expect(subject.brokers[0][:broker_port]).to eq(9092)
+      end
+
+      it "#topics returns data on our test topic" do
+        expect(subject.topics.length).to eq(1)
+        expect(subject.topics[0][:partition_count]).to eq(25)
+        expect(subject.topics[0][:partitions].length).to eq(25)
+        expect(subject.topics[0][:topic_name]).to eq(topic_name)
+      end
+    end
+  end
+
+  context "not passing in a topic name" do
+    subject { described_class.new(native_kafka, topic_name) }
+    let(:topic_name) { nil }
+    let(:test_topics) {
+      %w(consume_test_topic empty_test_topic load_test_topic produce_test_topic rake_test_topic watermarks_test_topic partitioner_test_topic)
+    } # Test topics crated in spec_helper.rb
+
+    it "#brokers returns our single broker" do
+      expect(subject.brokers.length).to eq(1)
+      expect(subject.brokers[0][:broker_id]).to eq(1)
+      expect(subject.brokers[0][:broker_name]).to eq("localhost")
+      expect(subject.brokers[0][:broker_port]).to eq(9092)
+    end
+
+    it "#topics returns data about all of our test topics" do
+      result = subject.topics.map { |topic| topic[:topic_name] }
+      expect(result).to include(*test_topics)
+    end
+  end
+
+  context "when a non-zero error code is returned" do
+    let(:topic_name) { SecureRandom.uuid.to_s }
+
+    before do
+      allow(Rdkafka::Bindings).to receive(:rd_kafka_metadata).and_return(-165)
+    end
+
+    it "creating the instance raises an exception" do
+      expect {
+        described_class.new(native_kafka, topic_name)
+      }.to raise_error(Rdkafka::RdkafkaError, /Local: Required feature not supported by broker \(unsupported_feature\)/)
+    end
+  end
+end

data/spec/rdkafka/producer/delivery_handle_spec.rb

@@ -12,42 +12,13 @@ describe Rdkafka::Producer::DeliveryHandle do
     end
   end
 
-  describe ".register and .remove" do
-    let(:pending_handle) { true }
-
-    it "should register and remove a delivery handle" do
-      Rdkafka::Producer::DeliveryHandle.register(subject.to_ptr.address, subject)
-      removed = Rdkafka::Producer::DeliveryHandle.remove(subject.to_ptr.address)
-      expect(removed).to eq subject
-      expect(Rdkafka::Producer::DeliveryHandle::REGISTRY).to be_empty
-    end
-  end
-
-  describe "#pending?" do
-    context "when true" do
-      let(:pending_handle) { true }
-
-      it "should be true" do
-        expect(subject.pending?).to be true
-      end
-    end
-
-    context "when not true" do
-      let(:pending_handle) { false }
-
-      it "should be false" do
-        expect(subject.pending?).to be false
-      end
-    end
-  end
-
   describe "#wait" do
     let(:pending_handle) { true }
 
     it "should wait until the timeout and then raise an error" do
       expect {
-        subject.wait(0.1)
-      }.to raise_error Rdkafka::Producer::DeliveryHandle::WaitTimeoutError
+        subject.wait(max_wait_timeout: 0.1)
+      }.to raise_error Rdkafka::Producer::DeliveryHandle::WaitTimeoutError, /delivery/
     end
 
     context "when not pending anymore and no error" do
@@ -61,22 +32,11 @@ describe Rdkafka::Producer::DeliveryHandle do
       end
 
       it "should wait without a timeout" do
-        report = subject.wait(nil)
+        report = subject.wait(max_wait_timeout: nil)
 
         expect(report.partition).to eq(2)
         expect(report.offset).to eq(100)
       end
     end
-
-    context "when not pending anymore and there was an error" do
-      let(:pending_handle) { false }
-      let(:response) { 20 }
-
-      it "should raise an rdkafka error" do
-        expect {
-          subject.wait
-        }.to raise_error Rdkafka::RdkafkaError
-      end
-    end
   end
 end

data/spec/rdkafka/producer/delivery_report_spec.rb

@@ -1,7 +1,7 @@
 require "spec_helper"
 
 describe Rdkafka::Producer::DeliveryReport do
-  subject { Rdkafka::Producer::DeliveryReport.new(2, 100) }
+  subject { Rdkafka::Producer::DeliveryReport.new(2, 100, "error") }
 
   it "should get the partition" do
     expect(subject.partition).to eq 2
@@ -10,4 +10,8 @@ describe Rdkafka::Producer::DeliveryReport do
   it "should get the offset" do
     expect(subject.offset).to eq 100
   end
+
+  it "should get the error" do
+    expect(subject.error).to eq "error"
+  end
 end

data/spec/rdkafka/producer_spec.rb

@@ -2,50 +2,102 @@ require "spec_helper"
 
 describe Rdkafka::Producer do
   let(:producer) { rdkafka_config.producer }
+  let(:consumer) { rdkafka_config.consumer }
 
   after do
     # Registry should always end up being empty
     expect(Rdkafka::Producer::DeliveryHandle::REGISTRY).to be_empty
+    producer.close
+    consumer.close
   end
 
   context "delivery callback" do
-    it "should set the callback" do
-      expect {
-        producer.delivery_callback = lambda do |delivery_handle|
-          puts stats
+    context "with a proc/lambda" do
+      it "should set the callback" do
+        expect {
+          producer.delivery_callback = lambda do |delivery_handle|
+            puts delivery_handle
+          end
+        }.not_to raise_error
+        expect(producer.delivery_callback).to respond_to :call
+      end
+
+      it "should call the callback when a message is delivered" do
+        @callback_called = false
+
+        producer.delivery_callback = lambda do |report|
+          expect(report).not_to be_nil
+          expect(report.partition).to eq 1
+          expect(report.offset).to be >= 0
+          @callback_called = true
         end
-      }.not_to raise_error
-      expect(producer.delivery_callback).to be_a Proc
-    end
 
-    it "should not accept a callback that's not a proc" do
-      expect {
-        producer.delivery_callback = 'a string'
-      }.to raise_error(TypeError)
-    end
+        # Produce a message
+        handle = producer.produce(
+          topic:   "produce_test_topic",
+          payload: "payload",
+          key:     "key"
+        )
+
+        # Wait for it to be delivered
+        handle.wait(max_wait_timeout: 15)
 
-    it "should call the callback when a message is delivered" do
-      @callback_called = false
+        # Join the producer thread.
+        producer.close
 
-      producer.delivery_callback = lambda do |report|
-        expect(report).not_to be_nil
-        expect(report.partition).to eq 1
-        expect(report.offset).to be >= 0
-        @callback_called = true
+        # Callback should have been called
+        expect(@callback_called).to be true
       end
+    end
 
-      # Produce a message
-      handle = producer.produce(
-        topic:   "produce_test_topic",
-        payload: "payload",
-        key:     "key"
-      )
+    context "with a callable object" do
+      it "should set the callback" do
+        callback = Class.new do
+          def call(stats); end
+        end
+        expect {
+          producer.delivery_callback = callback.new
+        }.not_to raise_error
+        expect(producer.delivery_callback).to respond_to :call
+      end
+
+      it "should call the callback when a message is delivered" do
+        called_report = []
+        callback = Class.new do
+          def initialize(called_report)
+            @called_report = called_report
+          end
+
+          def call(report)
+            @called_report << report
+          end
+        end
+        producer.delivery_callback = callback.new(called_report)
+
+        # Produce a message
+        handle = producer.produce(
+          topic:   "produce_test_topic",
+          payload: "payload",
+          key:     "key"
+        )
+
+        # Wait for it to be delivered
+        handle.wait(max_wait_timeout: 15)
 
-      # Wait for it to be delivered
-      handle.wait(5)
+        # Join the producer thread.
+        producer.close
 
-      # Callback should have been called
-      expect(@callback_called).to be true
+        # Callback should have been called
+        expect(called_report.first).not_to be_nil
+        expect(called_report.first.partition).to eq 1
+        expect(called_report.first.offset).to be >= 0
+      end
+    end
+
+    it "should not accept a callback that's not callable" do
+      expect {
+        producer.delivery_callback = 'a string'
+      }.to raise_error(TypeError)
     end
   end
 
@@ -55,7 +107,7 @@ describe Rdkafka::Producer do
         payload: "payload",
         key:     "key"
      )
-    }.to raise_error ArgumentError, "missing keyword: topic"
+    }.to raise_error ArgumentError, /missing keyword: [\:]?topic/
   end
 
   it "should produce a message" do
@@ -70,7 +122,7 @@ describe Rdkafka::Producer do
     expect(handle.pending?).to be true
 
     # Check delivery handle and report
-    report = handle.wait(5)
+    report = handle.wait(max_wait_timeout: 5)
     expect(handle.pending?).to be false
     expect(report).not_to be_nil
     expect(report.partition).to eq 1
@@ -82,14 +134,15 @@ describe Rdkafka::Producer do
     # Consume message and verify it's content
     message = wait_for_message(
       topic: "produce_test_topic",
-      delivery_report: report
+      delivery_report: report,
+      consumer: consumer
     )
     expect(message.partition).to eq 1
     expect(message.payload).to eq "payload"
     expect(message.key).to eq "key"
     # Since api.version.request is on by default we will get
     # the message creation timestamp if it's not set.
-    expect(message.timestamp).to be_within(5).of(Time.now)
+    expect(message.timestamp).to be_within(10).of(Time.now)
   end
 
   it "should produce a message with a specified partition" do
@@ -100,29 +153,65 @@ describe Rdkafka::Producer do
       key:       "key partition",
       partition: 1
     )
-    report = handle.wait(5)
+    report = handle.wait(max_wait_timeout: 5)
 
     # Consume message and verify it's content
     message = wait_for_message(
       topic: "produce_test_topic",
-      delivery_report: report
+      delivery_report: report,
+      consumer: consumer
     )
     expect(message.partition).to eq 1
     expect(message.key).to eq "key partition"
   end
 
+  it "should produce a message to the same partition with a similar partition key" do
+    # Avoid partitioner collisions.
+    while true
+      key = ('a'..'z').to_a.shuffle.take(10).join('')
+      partition_key = ('a'..'z').to_a.shuffle.take(10).join('')
+      partition_count = producer.partition_count('partitioner_test_topic')
+      break if (Zlib.crc32(key) % partition_count) != (Zlib.crc32(partition_key) % partition_count)
+    end
+
+    # Produce a message with key, partition_key and key + partition_key
+    messages = [{key: key}, {partition_key: partition_key}, {key: key, partition_key: partition_key}]
+
+    messages = messages.map do |m|
+      handle = producer.produce(
+        topic:     "partitioner_test_topic",
+        payload:   "payload partition",
+        key:       m[:key],
+        partition_key: m[:partition_key]
+      )
+      report = handle.wait(max_wait_timeout: 5)
+
+      wait_for_message(
+        topic: "partitioner_test_topic",
+        delivery_report: report,
+      )
+    end
+
+    expect(messages[0].partition).not_to eq(messages[2].partition)
+    expect(messages[1].partition).to eq(messages[2].partition)
+    expect(messages[0].key).to eq key
+    expect(messages[1].key).to be_nil
+    expect(messages[2].key).to eq key
+  end
+
   it "should produce a message with utf-8 encoding" do
     handle = producer.produce(
       topic:   "produce_test_topic",
       payload: "Τη γλώσσα μου έδωσαν ελληνική",
       key:     "key utf8"
     )
-    report = handle.wait(5)
+    report = handle.wait(max_wait_timeout: 5)
 
     # Consume message and verify it's content
     message = wait_for_message(
       topic: "produce_test_topic",
-      delivery_report: report
+      delivery_report: report,
+      consumer: consumer
     )
 
     expect(message.partition).to eq 1
@@ -149,12 +238,13 @@ describe Rdkafka::Producer do
         key:       "key timestamp",
         timestamp: 1505069646252
       )
-      report = handle.wait(5)
+      report = handle.wait(max_wait_timeout: 5)
 
       # Consume message and verify it's content
       message = wait_for_message(
         topic: "produce_test_topic",
-        delivery_report: report
+        delivery_report: report,
+        consumer: consumer
       )
 
       expect(message.partition).to eq 2
@@ -169,12 +259,13 @@ describe Rdkafka::Producer do
         key:       "key timestamp",
         timestamp: Time.at(1505069646, 353_000)
       )
-      report = handle.wait(5)
+      report = handle.wait(max_wait_timeout: 5)
 
       # Consume message and verify it's content
       message = wait_for_message(
         topic: "produce_test_topic",
-        delivery_report: report
+        delivery_report: report,
+        consumer: consumer
       )
 
       expect(message.partition).to eq 2
@@ -188,12 +279,13 @@ describe Rdkafka::Producer do
       topic:   "produce_test_topic",
       payload: "payload no key"
     )
-    report = handle.wait(5)
+    report = handle.wait(max_wait_timeout: 5)
 
     # Consume message and verify it's content
     message = wait_for_message(
       topic: "produce_test_topic",
-      delivery_report: report
+      delivery_report: report,
+      consumer: consumer
     )
 
     expect(message.key).to be_nil
@@ -205,12 +297,13 @@ describe Rdkafka::Producer do
       topic: "produce_test_topic",
       key:   "key no payload"
     )
-    report = handle.wait(5)
+    report = handle.wait(max_wait_timeout: 5)
 
     # Consume message and verify it's content
     message = wait_for_message(
       topic: "produce_test_topic",
-      delivery_report: report
+      delivery_report: report,
+      consumer: consumer
     )
 
     expect(message.key).to eq "key no payload"
@@ -224,12 +317,13 @@ describe Rdkafka::Producer do
       key:       "key headers",
       headers:   { foo: :bar, baz: :foobar }
     )
-    report = handle.wait(5)
+    report = handle.wait(max_wait_timeout: 5)
 
     # Consume message and verify it's content
     message = wait_for_message(
       topic: "produce_test_topic",
-      delivery_report: report
+      delivery_report: report,
+      consumer: consumer
     )
 
     expect(message.payload).to eq "payload headers"
@@ -246,12 +340,13 @@ describe Rdkafka::Producer do
       key:       "key headers",
       headers:   {}
     )
-    report = handle.wait(5)
+    report = handle.wait(max_wait_timeout: 5)
 
     # Consume message and verify it's content
     message = wait_for_message(
       topic: "produce_test_topic",
-      delivery_report: report
+      delivery_report: report,
+      consumer: consumer
     )
 
     expect(message.payload).to eq "payload headers"
@@ -280,55 +375,58 @@ describe Rdkafka::Producer do
     end
   end
 
-# TODO this spec crashes if you create and use the producer before
-# forking like so:
-#
-# @producer = producer
-#
-# This will be added as part of https://github.com/appsignal/rdkafka-ruby/issues/19
-  #it "should produce a message in a forked process" do
-  #  # Fork, produce a message, send the report of a pipe and
-  #  # wait for it in the main process.
-
-  #  reader, writer = IO.pipe
-
-  #  fork do
-  #    reader.close
-
-  #    handle = producer.produce(
-  #      topic:   "produce_test_topic",
-  #      payload: "payload",
-  #      key:     "key"
-  #    )
-
-  #    report = handle.wait(5)
-  #    producer.close
-
-  #    report_json = JSON.generate(
-  #      "partition" => report.partition,
-  #      "offset" => report.offset
-  #    )
-
-  #    writer.write(report_json)
-  #  end
-
-  #  writer.close
-
-  #  report_hash = JSON.parse(reader.read)
-  #  report = Rdkafka::Producer::DeliveryReport.new(
-  #    report_hash["partition"],
-  #    report_hash["offset"]
-  #  )
-
-  #  # Consume message and verify it's content
-  #  message = wait_for_message(
-  #    topic: "produce_test_topic",
-  #    delivery_report: report
-  #  )
-  #  expect(message.partition).to eq 1
-  #  expect(message.payload).to eq "payload"
-  #  expect(message.key).to eq "key"
-  #end
+  it "should produce a message in a forked process" do
+    # Fork, produce a message, send the report over a pipe and
+    # wait for and check the message in the main process.
+
+    # Kernel#fork is not available in JRuby
+    skip if defined?(JRUBY_VERSION)
+
+    reader, writer = IO.pipe
+
+    fork do
+      reader.close
+
+      # Avoids sharing the socket between processes.
+      producer = rdkafka_config.producer
+
+      handle = producer.produce(
+        topic:   "produce_test_topic",
+        payload: "payload-forked",
+        key:     "key-forked"
+      )
+
+      report = handle.wait(max_wait_timeout: 5)
+
+      report_json = JSON.generate(
+        "partition" => report.partition,
+        "offset" => report.offset
+      )
+
+      writer.write(report_json)
+      writer.close
+      producer.close
+    end
+
+    writer.close
+    report_hash = JSON.parse(reader.read)
+    report = Rdkafka::Producer::DeliveryReport.new(
+      report_hash["partition"],
+      report_hash["offset"]
+    )
+
+    reader.close
+
+    # Consume message and verify it's content
+    message = wait_for_message(
+      topic: "produce_test_topic",
+      delivery_report: report,
+      consumer: consumer
+    )
+    expect(message.partition).to eq 0
+    expect(message.payload).to eq "payload-forked"
+    expect(message.key).to eq "key-forked"
+  end
 
   it "should raise an error when producing fails" do
     expect(Rdkafka::Bindings).to receive(:rd_kafka_producev).and_return(20)
@@ -348,10 +446,32 @@ describe Rdkafka::Producer do
       key:     "key timeout"
     )
     expect {
-      handle.wait(0)
+      handle.wait(max_wait_timeout: 0)
     }.to raise_error Rdkafka::Producer::DeliveryHandle::WaitTimeoutError
 
     # Waiting a second time should work
-    handle.wait(5)
+    handle.wait(max_wait_timeout: 5)
+  end
+
+  context "methods that should not be called after a producer has been closed" do
+    before do
+      producer.close
+    end
+
+    # Affected methods and a non-invalid set of parameters for the method
+    {
+        :produce         => { topic: nil },
+        :partition_count => nil,
+    }.each do |method, args|
+      it "raises an exception if #{method} is called" do
+        expect {
+          if args.is_a?(Hash)
+            producer.public_send(method, **args)
+          else
+            producer.public_send(method, args)
+          end
+        }.to raise_exception(Rdkafka::ClosedProducerError, /#{method.to_s}/)
+      end
+    end
   end
 end