rdkafka 0.8.0.beta.1 → 0.10.0

data/spec/rdkafka/consumer_spec.rb

@@ -1,5 +1,6 @@
 require "spec_helper"
 require "ostruct"
+require 'securerandom'
 
 describe Rdkafka::Consumer do
   let(:config) { rdkafka_config }
@@ -271,8 +272,18 @@ describe Rdkafka::Consumer do
   describe "#close" do
     it "should close a consumer" do
       consumer.subscribe("consume_test_topic")
+      100.times do |i|
+        report = producer.produce(
+          topic:     "consume_test_topic",
+          payload:   "payload #{i}",
+          key:       "key #{i}",
+          partition: 0
+        ).wait
+      end
       consumer.close
-      expect(consumer.poll(100)).to be_nil
+      expect {
+        consumer.poll(100)
+      }.to raise_error(Rdkafka::ClosedConsumerError, /poll/)
     end
   end
 
@@ -662,7 +673,230 @@ describe Rdkafka::Consumer do
       # should break the each loop.
       consumer.each_with_index do |message, i|
         expect(message).to be_a Rdkafka::Consumer::Message
-        consumer.close if i == 10
+        break if i == 10
+      end
+      consumer.close
+    end
+  end
+
+  describe "#each_batch" do
+    let(:message_payload) { 'a' * 10 }
+
+    before do
+      @topic = SecureRandom.base64(10).tr('+=/', '')
+    end
+
+    after do
+      @topic = nil
+    end
+
+    def topic_name
+      @topic
+    end
+
+    def produce_n(n)
+      handles = []
+      n.times do |i|
+        handles << producer.produce(
+          topic:     topic_name,
+          payload:   Time.new.to_f.to_s,
+          key:       i.to_s,
+          partition: 0
+        )
+      end
+      handles.each(&:wait)
+    end
+
+    def new_message
+      instance_double("Rdkafka::Consumer::Message").tap do |message|
+        allow(message).to receive(:payload).and_return(message_payload)
+      end
+    end
+
+    it "retrieves messages produced into a topic" do
+      # This is the only each_batch test that actually produces real messages
+      # into a topic in the real kafka of the container.
+      #
+      # The other tests stub 'poll' which makes them faster and more reliable,
+      # but it makes sense to keep a single test with a fully integrated flow.
+      # This will help to catch breaking changes in the behavior of 'poll',
+      # libdrkafka, or Kafka.
+      #
+      # This is, in effect, an integration test and the subsequent specs are
+      # unit tests.
+      create_topic_handle = rdkafka_config.admin.create_topic(topic_name, 1, 1)
+      create_topic_handle.wait(max_wait_timeout: 15.0)
+      consumer.subscribe(topic_name)
+      produce_n 42
+      all_yields = []
+      consumer.each_batch(max_items: 10) do |batch|
+        all_yields << batch
+        break if all_yields.flatten.size >= 42
+      end
+      expect(all_yields.flatten.first).to be_a Rdkafka::Consumer::Message
+      expect(all_yields.flatten.size).to eq 42
+      expect(all_yields.size).to be > 4
+      expect(all_yields.flatten.map(&:key)).to eq (0..41).map { |x| x.to_s }
+    end
+
+    it "should batch poll results and yield arrays of messages" do
+      consumer.subscribe(topic_name)
+      all_yields = []
+      expect(consumer)
+        .to receive(:poll)
+        .exactly(10).times
+        .and_return(new_message)
+      consumer.each_batch(max_items: 10) do |batch|
+        all_yields << batch
+        break if all_yields.flatten.size >= 10
+      end
+      expect(all_yields.first).to be_instance_of(Array)
+      expect(all_yields.flatten.size).to eq 10
+      non_empty_yields = all_yields.reject { |batch| batch.empty? }
+      expect(non_empty_yields.size).to be < 10
+    end
+
+    it "should yield a partial batch if the timeout is hit with some messages" do
+      consumer.subscribe(topic_name)
+      poll_count = 0
+      expect(consumer)
+        .to receive(:poll)
+        .at_least(3).times do
+        poll_count = poll_count + 1
+        if poll_count > 2
+          sleep 0.1
+          nil
+        else
+          new_message
+        end
+      end
+      all_yields = []
+      consumer.each_batch(max_items: 10) do |batch|
+        all_yields << batch
+        break if all_yields.flatten.size >= 2
+      end
+      expect(all_yields.flatten.size).to eq 2
+    end
+
+    it "should yield [] if nothing is received before the timeout" do
+      create_topic_handle = rdkafka_config.admin.create_topic(topic_name, 1, 1)
+      create_topic_handle.wait(max_wait_timeout: 15.0)
+      consumer.subscribe(topic_name)
+      consumer.each_batch do |batch|
+        expect(batch).to eq([])
+        break
+      end
+    end
+
+    it "should yield batchs of max_items in size if messages are already fetched" do
+      yielded_batches = []
+      expect(consumer)
+        .to receive(:poll)
+        .with(anything)
+        .exactly(20).times
+        .and_return(new_message)
+
+      consumer.each_batch(max_items: 10, timeout_ms: 500) do |batch|
+        yielded_batches << batch
+        break if yielded_batches.flatten.size >= 20
+        break if yielded_batches.size >= 20 # so failure doesn't hang
+      end
+      expect(yielded_batches.size).to eq 2
+      expect(yielded_batches.map(&:size)).to eq 2.times.map { 10 }
+    end
+
+    it "should yield batchs as soon as bytes_threshold is hit" do
+      yielded_batches = []
+      expect(consumer)
+        .to receive(:poll)
+        .with(anything)
+        .exactly(20).times
+        .and_return(new_message)
+
+      consumer.each_batch(bytes_threshold: message_payload.size * 4, timeout_ms: 500) do |batch|
+        yielded_batches << batch
+        break if yielded_batches.flatten.size >= 20
+        break if yielded_batches.size >= 20 # so failure doesn't hang
+      end
+      expect(yielded_batches.size).to eq 5
+      expect(yielded_batches.map(&:size)).to eq 5.times.map { 4 }
+    end
+
+    context "error raised from poll and yield_on_error is true" do
+      it "should yield buffered exceptions on rebalance, then break" do
+        config = rdkafka_config({:"enable.auto.commit" => false,
+                                 :"enable.auto.offset.store" => false })
+        consumer = config.consumer
+        consumer.subscribe(topic_name)
+        loop_count = 0
+        batches_yielded = []
+        exceptions_yielded = []
+        each_batch_iterations = 0
+        poll_count = 0
+        expect(consumer)
+          .to receive(:poll)
+          .with(anything)
+          .exactly(3).times
+          .and_wrap_original do |method, *args|
+              poll_count = poll_count + 1
+              if poll_count == 3
+                raise Rdkafka::RdkafkaError.new(27,
+                    "partitions ... too ... heavy ... must ... rebalance")
+              else
+                new_message
+              end
+          end
+        expect {
+          consumer.each_batch(max_items: 30, yield_on_error: true) do |batch, pending_error|
+            batches_yielded << batch
+            exceptions_yielded << pending_error
+            each_batch_iterations = each_batch_iterations + 1
+          end
+        }.to raise_error(Rdkafka::RdkafkaError)
+        expect(poll_count).to eq 3
+        expect(each_batch_iterations).to eq 1
+        expect(batches_yielded.size).to eq 1
+        expect(batches_yielded.first.size).to eq 2
+        expect(exceptions_yielded.flatten.size).to eq 1
+        expect(exceptions_yielded.flatten.first).to be_instance_of(Rdkafka::RdkafkaError)
+      end
+    end
+
+    context "error raised from poll and yield_on_error is false" do
+      it "should yield buffered exceptions on rebalance, then break" do
+        config = rdkafka_config({:"enable.auto.commit" => false,
+                                 :"enable.auto.offset.store" => false })
+        consumer = config.consumer
+        consumer.subscribe(topic_name)
+        loop_count = 0
+        batches_yielded = []
+        exceptions_yielded = []
+        each_batch_iterations = 0
+        poll_count = 0
+        expect(consumer)
+          .to receive(:poll)
+          .with(anything)
+          .exactly(3).times
+          .and_wrap_original do |method, *args|
+              poll_count = poll_count + 1
+              if poll_count == 3
+                raise Rdkafka::RdkafkaError.new(27,
+                    "partitions ... too ... heavy ... must ... rebalance")
+              else
+                new_message
+              end
+          end
+        expect {
+          consumer.each_batch(max_items: 30, yield_on_error: false) do |batch, pending_error|
+            batches_yielded << batch
+            exceptions_yielded << pending_error
+            each_batch_iterations = each_batch_iterations + 1
+          end
+        }.to raise_error(Rdkafka::RdkafkaError)
+        expect(poll_count).to eq 3
+        expect(each_batch_iterations).to eq 0
+        expect(batches_yielded.size).to eq 0
+        expect(exceptions_yielded.size).to eq 0
       end
     end
   end
@@ -723,4 +957,34 @@ describe Rdkafka::Consumer do
       consumer.close
     end
   end
+
+  context "methods that should not be called after a consumer has been closed" do
+    before do
+      consumer.close
+    end
+
+    # Affected methods and a non-invalid set of parameters for the method
+    {
+        :subscribe               => [ nil ],
+        :unsubscribe             => nil,
+        :each_batch              => nil,
+        :pause                   => [ nil ],
+        :resume                  => [ nil ],
+        :subscription            => nil,
+        :assign                  => [ nil ],
+        :assignment              => nil,
+        :committed               => [],
+        :query_watermark_offsets => [ nil, nil ],
+    }.each do |method, args|
+      it "raises an exception if #{method} is called" do
+        expect {
+          if args.nil?
+            consumer.public_send(method)
+          else
+            consumer.public_send(method, *args)
+          end
+        }.to raise_exception(Rdkafka::ClosedConsumerError, /#{method.to_s}/)
+      end
+    end
+  end
 end

data/spec/rdkafka/error_spec.rb

@@ -11,6 +11,10 @@ describe Rdkafka::RdkafkaError do
     expect(Rdkafka::RdkafkaError.new(10, "message prefix").message_prefix).to eq "message prefix"
   end
 
+  it "should create an error with a broker message" do
+    expect(Rdkafka::RdkafkaError.new(10, broker_message: "broker message").broker_message).to eq "broker message"
+  end
+
   describe "#code" do
     it "should handle an invalid response" do
       expect(Rdkafka::RdkafkaError.new(933975).code).to eq :err_933975?

data/spec/rdkafka/metadata_spec.rb

@@ -0,0 +1,78 @@
+require "spec_helper"
+require "securerandom"
+
+describe Rdkafka::Metadata do
+  let(:config)        { rdkafka_config }
+  let(:native_config) { config.send(:native_config) }
+  let(:native_kafka)  { config.send(:native_kafka, native_config, :rd_kafka_consumer) }
+
+  after do
+    Rdkafka::Bindings.rd_kafka_consumer_close(native_kafka)
+    Rdkafka::Bindings.rd_kafka_destroy(native_kafka)
+  end
+
+  context "passing in a topic name" do
+    context "that is non-existent topic" do
+      let(:topic_name) { SecureRandom.uuid.to_s }
+
+      it "raises an appropriate exception" do
+        expect {
+          described_class.new(native_kafka, topic_name)
+        }.to raise_exception(Rdkafka::RdkafkaError, "Broker: Unknown topic or partition (unknown_topic_or_part)")
+      end
+    end
+
+    context "that is one of our test topics" do
+      subject { described_class.new(native_kafka, topic_name) }
+      let(:topic_name) { "partitioner_test_topic" }
+
+      it "#brokers returns our single broker" do
+        expect(subject.brokers.length).to eq(1)
+        expect(subject.brokers[0][:broker_id]).to eq(1)
+        expect(subject.brokers[0][:broker_name]).to eq("localhost")
+        expect(subject.brokers[0][:broker_port]).to eq(9092)
+      end
+
+      it "#topics returns data on our test topic" do
+        expect(subject.topics.length).to eq(1)
+        expect(subject.topics[0][:partition_count]).to eq(25)
+        expect(subject.topics[0][:partitions].length).to eq(25)
+        expect(subject.topics[0][:topic_name]).to eq(topic_name)
+      end
+    end
+  end
+
+  context "not passing in a topic name" do
+    subject { described_class.new(native_kafka, topic_name) }
+    let(:topic_name) { nil }
+    let(:test_topics) {
+      %w(consume_test_topic empty_test_topic load_test_topic produce_test_topic rake_test_topic watermarks_test_topic partitioner_test_topic)
+    } # Test topics crated in spec_helper.rb
+
+    it "#brokers returns our single broker" do
+      expect(subject.brokers.length).to eq(1)
+      expect(subject.brokers[0][:broker_id]).to eq(1)
+      expect(subject.brokers[0][:broker_name]).to eq("localhost")
+      expect(subject.brokers[0][:broker_port]).to eq(9092)
+    end
+
+    it "#topics returns data about all of our test topics" do
+      result = subject.topics.map { |topic| topic[:topic_name] }
+      expect(result).to include(*test_topics)
+    end
+  end
+
+  context "when a non-zero error code is returned" do
+    let(:topic_name) { SecureRandom.uuid.to_s }
+
+    before do
+      allow(Rdkafka::Bindings).to receive(:rd_kafka_metadata).and_return(-165)
+    end
+
+    it "creating the instance raises an exception" do
+      expect {
+        described_class.new(native_kafka, topic_name)
+      }.to raise_error(Rdkafka::RdkafkaError, /Local: Required feature not supported by broker \(unsupported_feature\)/)
+    end
+  end
+end

data/spec/rdkafka/producer/delivery_handle_spec.rb

@@ -12,42 +12,13 @@ describe Rdkafka::Producer::DeliveryHandle do
     end
   end
 
-  describe ".register and .remove" do
-    let(:pending_handle) { true }
-
-    it "should register and remove a delivery handle" do
-      Rdkafka::Producer::DeliveryHandle.register(subject.to_ptr.address, subject)
-      removed = Rdkafka::Producer::DeliveryHandle.remove(subject.to_ptr.address)
-      expect(removed).to eq subject
-      expect(Rdkafka::Producer::DeliveryHandle::REGISTRY).to be_empty
-    end
-  end
-
-  describe "#pending?" do
-    context "when true" do
-      let(:pending_handle) { true }
-
-      it "should be true" do
-        expect(subject.pending?).to be true
-      end
-    end
-
-    context "when not true" do
-      let(:pending_handle) { false }
-
-      it "should be false" do
-        expect(subject.pending?).to be false
-      end
-    end
-  end
-
   describe "#wait" do
     let(:pending_handle) { true }
 
     it "should wait until the timeout and then raise an error" do
       expect {
         subject.wait(max_wait_timeout: 0.1)
-      }.to raise_error Rdkafka::Producer::DeliveryHandle::WaitTimeoutError
+      }.to raise_error Rdkafka::Producer::DeliveryHandle::WaitTimeoutError, /delivery/
     end
 
     context "when not pending anymore and no error" do
@@ -67,16 +38,5 @@ describe Rdkafka::Producer::DeliveryHandle do
         expect(report.offset).to eq(100)
       end
     end
-
-    context "when not pending anymore and there was an error" do
-      let(:pending_handle) { false }
-      let(:response) { 20 }
-
-      it "should raise an rdkafka error" do
-        expect {
-          subject.wait
-        }.to raise_error Rdkafka::RdkafkaError
-      end
-    end
   end
 end

data/spec/rdkafka/producer_spec.rb

@@ -12,47 +12,92 @@ describe Rdkafka::Producer do
   end
 
   context "delivery callback" do
-    it "should set the callback" do
-      expect {
-        producer.delivery_callback = lambda do |delivery_handle|
-          puts stats
+    context "with a proc/lambda" do
+      it "should set the callback" do
+        expect {
+          producer.delivery_callback = lambda do |delivery_handle|
+            puts delivery_handle
+          end
+        }.not_to raise_error
+        expect(producer.delivery_callback).to respond_to :call
+      end
+
+      it "should call the callback when a message is delivered" do
+        @callback_called = false
+
+        producer.delivery_callback = lambda do |report|
+          expect(report).not_to be_nil
+          expect(report.partition).to eq 1
+          expect(report.offset).to be >= 0
+          @callback_called = true
         end
-      }.not_to raise_error
-      expect(producer.delivery_callback).to be_a Proc
-    end
 
-    it "should not accept a callback that's not a proc" do
-      expect {
-        producer.delivery_callback = 'a string'
-      }.to raise_error(TypeError)
-    end
+        # Produce a message
+        handle = producer.produce(
+          topic:   "produce_test_topic",
+          payload: "payload",
+          key:     "key"
+        )
 
-    it "should call the callback when a message is delivered" do
-      @callback_called = false
+        # Wait for it to be delivered
+        handle.wait(max_wait_timeout: 15)
 
+        # Join the producer thread.
+        producer.close
 
-      producer.delivery_callback = lambda do |report|
-        expect(report).not_to be_nil
-        expect(report.partition).to eq 1
-        expect(report.offset).to be >= 0
-        @callback_called = true
+        # Callback should have been called
+        expect(@callback_called).to be true
       end
+    end
 
-      # Produce a message
-      handle = producer.produce(
-        topic:   "produce_test_topic",
-        payload: "payload",
-        key:     "key"
-      )
+    context "with a callable object" do
+      it "should set the callback" do
+        callback = Class.new do
+          def call(stats); end
+        end
+        expect {
+          producer.delivery_callback = callback.new
+        }.not_to raise_error
+        expect(producer.delivery_callback).to respond_to :call
+      end
 
-      # Wait for it to be delivered
-      handle.wait(max_wait_timeout: 15)
+      it "should call the callback when a message is delivered" do
+        called_report = []
+        callback = Class.new do
+          def initialize(called_report)
+            @called_report = called_report
+          end
 
-      # Join the producer thread.
-      producer.close
+          def call(report)
+            @called_report << report
+          end
+        end
+        producer.delivery_callback = callback.new(called_report)
+
+        # Produce a message
+        handle = producer.produce(
+          topic:   "produce_test_topic",
+          payload: "payload",
+          key:     "key"
+        )
+
+        # Wait for it to be delivered
+        handle.wait(max_wait_timeout: 15)
 
-      # Callback should have been called
-      expect(@callback_called).to be true
+        # Join the producer thread.
+        producer.close
+
+        # Callback should have been called
+        expect(called_report.first).not_to be_nil
+        expect(called_report.first.partition).to eq 1
+        expect(called_report.first.offset).to be >= 0
+      end
+    end
+
+    it "should not accept a callback that's not callable" do
+      expect {
+        producer.delivery_callback = 'a string'
+      }.to raise_error(TypeError)
     end
   end
 
@@ -407,4 +452,26 @@ describe Rdkafka::Producer do
     # Waiting a second time should work
     handle.wait(max_wait_timeout: 5)
   end
+
+  context "methods that should not be called after a producer has been closed" do
+    before do
+      producer.close
+    end
+
+    # Affected methods and a non-invalid set of parameters for the method
+    {
+        :produce         => { topic: nil },
+        :partition_count => nil,
+    }.each do |method, args|
+      it "raises an exception if #{method} is called" do
+        expect {
+          if args.is_a?(Hash)
+            producer.public_send(method, **args)
+          else
+            producer.public_send(method, args)
+          end
+        }.to raise_exception(Rdkafka::ClosedProducerError, /#{method.to_s}/)
+      end
+    end
+  end
 end