rdkafka 0.6.0 → 0.9.0

data/spec/rdkafka/bindings_spec.rb

@@ -1,4 +1,5 @@
 require "spec_helper"
+require 'zlib'
 
 describe Rdkafka::Bindings do
   it "should load librdkafka" do
@@ -7,12 +8,12 @@ describe Rdkafka::Bindings do
 
   describe ".lib_extension" do
     it "should know the lib extension for darwin" do
-      expect(Gem::Platform.local).to receive(:os).and_return("darwin-aaa")
+      stub_const('RbConfig::CONFIG', 'host_os' =>'darwin')
       expect(Rdkafka::Bindings.lib_extension).to eq "dylib"
     end
 
     it "should know the lib extension for linux" do
-      expect(Gem::Platform.local).to receive(:os).and_return("linux")
+      stub_const('RbConfig::CONFIG', 'host_os' =>'linux')
       expect(Rdkafka::Bindings.lib_extension).to eq "so"
     end
   end
@@ -24,39 +25,56 @@ describe Rdkafka::Bindings do
   end
 
   describe "log callback" do
-    let(:log) { StringIO.new }
+    let(:log_queue) { Rdkafka::Config.log_queue }
     before do
-      Rdkafka::Config.logger = Logger.new(log)
+      allow(log_queue).to receive(:<<)
     end
 
     it "should log fatal messages" do
       Rdkafka::Bindings::LogCallback.call(nil, 0, nil, "log line")
-      expect(log.string).to include "FATAL -- : rdkafka: log line"
+      expect(log_queue).to have_received(:<<).with([Logger::FATAL, "rdkafka: log line"])
     end
 
     it "should log error messages" do
       Rdkafka::Bindings::LogCallback.call(nil, 3, nil, "log line")
-      expect(log.string).to include "ERROR -- : rdkafka: log line"
+      expect(log_queue).to have_received(:<<).with([Logger::ERROR, "rdkafka: log line"])
     end
 
     it "should log warning messages" do
       Rdkafka::Bindings::LogCallback.call(nil, 4, nil, "log line")
-      expect(log.string).to include "WARN -- : rdkafka: log line"
+      expect(log_queue).to have_received(:<<).with([Logger::WARN, "rdkafka: log line"])
     end
 
     it "should log info messages" do
       Rdkafka::Bindings::LogCallback.call(nil, 5, nil, "log line")
-      expect(log.string).to include "INFO -- : rdkafka: log line"
+      expect(log_queue).to have_received(:<<).with([Logger::INFO, "rdkafka: log line"])
     end
 
     it "should log debug messages" do
       Rdkafka::Bindings::LogCallback.call(nil, 7, nil, "log line")
-      expect(log.string).to include "DEBUG -- : rdkafka: log line"
+      expect(log_queue).to have_received(:<<).with([Logger::DEBUG, "rdkafka: log line"])
     end
 
     it "should log unknown messages" do
       Rdkafka::Bindings::LogCallback.call(nil, 100, nil, "log line")
-      expect(log.string).to include "ANY -- : rdkafka: log line"
+      expect(log_queue).to have_received(:<<).with([Logger::UNKNOWN, "rdkafka: log line"])
+    end
+  end
+
+  describe "partitioner" do
+    let(:partition_key) { ('a'..'z').to_a.shuffle.take(15).join('') }
+    let(:partition_count) { rand(50) + 1 }
+
+    it "should return the same partition for a similar string and the same partition count" do
+      result_1 = Rdkafka::Bindings.partitioner(partition_key, partition_count)
+      result_2 = Rdkafka::Bindings.partitioner(partition_key, partition_count)
+      expect(result_1).to eq(result_2)
+    end
+
+    it "should match the old partitioner" do
+      result_1 = Rdkafka::Bindings.partitioner(partition_key, partition_count)
+      result_2 = (Zlib.crc32(partition_key) % partition_count)
+      expect(result_1).to eq(result_2)
     end
   end
 

data/spec/rdkafka/callbacks_spec.rb

@@ -0,0 +1,20 @@
+require "spec_helper"
+
+describe Rdkafka::Callbacks do
+
+  # The code in the call back functions is 100% covered by other specs.  Due to
+  # the large number of collaborators, and the fact that FFI does not play
+  # nicely with doubles, it was very difficult to construct tests that were
+  # not over-mocked.
+
+  # For debugging purposes, if you suspect that you are running into trouble in
+  # one of the callback functions, it may be helpful to surround the inner body
+  # of the method with something like:
+  #
+  #   begin
+  #     <method body>
+  #   rescue => ex; puts ex.inspect; puts ex.backtrace; end;
+  #
+  # This will output to STDOUT any exceptions that are being raised in the callback.
+
+end

data/spec/rdkafka/config_spec.rb

@@ -18,19 +18,46 @@ describe Rdkafka::Config do
         Rdkafka::Config.logger = nil
       }.to raise_error(Rdkafka::Config::NoLoggerError)
     end
+
+    it "supports logging queue" do
+      log = StringIO.new
+      Rdkafka::Config.logger = Logger.new(log)
+
+      Rdkafka::Config.log_queue << [Logger::FATAL, "I love testing"]
+      20.times do
+        break if log.string != ""
+        sleep 0.05
+      end
+
+      expect(log.string).to include "FATAL -- : I love testing"
+    end
   end
 
   context "statistics callback" do
-    it "should set the callback" do
-      expect {
-        Rdkafka::Config.statistics_callback = lambda do |stats|
-          puts stats
+    context "with a proc/lambda" do
+      it "should set the callback" do
+        expect {
+          Rdkafka::Config.statistics_callback = lambda do |stats|
+            puts stats
+          end
+        }.not_to raise_error
+        expect(Rdkafka::Config.statistics_callback).to respond_to :call
+      end
+    end
+
+    context "with a callable object" do
+      it "should set the callback" do
+        callback = Class.new do
+          def call(stats); end
         end
-      }.not_to raise_error
-      expect(Rdkafka::Config.statistics_callback).to be_a Proc
+        expect {
+          Rdkafka::Config.statistics_callback = callback.new
+        }.not_to raise_error
+        expect(Rdkafka::Config.statistics_callback).to respond_to :call
+      end
     end
 
-    it "should not accept a callback that's not a proc" do
+    it "should not accept a callback that's not callable" do
       expect {
         Rdkafka::Config.statistics_callback = 'a string'
       }.to raise_error(TypeError)
@@ -50,7 +77,9 @@ describe Rdkafka::Config do
     end
 
     it "should create a consumer with valid config" do
-      expect(rdkafka_config.consumer).to be_a Rdkafka::Consumer
+      consumer = rdkafka_config.consumer
+      expect(consumer).to be_a Rdkafka::Consumer
+      consumer.close
     end
 
     it "should raise an error when creating a consumer with invalid config" do
@@ -76,7 +105,9 @@ describe Rdkafka::Config do
     end
 
     it "should create a producer with valid config" do
-      expect(rdkafka_config.producer).to be_a Rdkafka::Producer
+      producer = rdkafka_config.producer
+      expect(producer).to be_a Rdkafka::Producer
+      producer.close
     end
 
     it "should raise an error when creating a producer with invalid config" do
@@ -86,6 +117,17 @@ describe Rdkafka::Config do
       }.to raise_error(Rdkafka::Config::ConfigError, "No such configuration property: \"invalid.key\"")
     end
 
+    it "should allow configuring zstd compression" do
+      config = Rdkafka::Config.new('compression.codec' => 'zstd')
+      begin
+        expect(config.producer).to be_a Rdkafka::Producer
+        config.producer.close
+      rescue Rdkafka::Config::ConfigError => ex
+        pending "Zstd compression not supported on this machine"
+        raise ex
+      end
+    end
+
     it "should raise an error when client creation fails for a consumer" do
       config = Rdkafka::Config.new(
         "security.protocol" => "SSL",

data/spec/rdkafka/consumer/message_spec.rb

@@ -1,7 +1,8 @@
 require "spec_helper"
 
 describe Rdkafka::Consumer::Message do
-  let(:native_topic) { new_native_topic }
+  let(:native_client) { new_native_client }
+  let(:native_topic) { new_native_topic(native_client: native_client) }
   let(:payload) { nil }
   let(:key) { nil }
   let(:native_message) do
@@ -24,6 +25,10 @@ describe Rdkafka::Consumer::Message do
     end
   end
 
+  after(:each) do
+    Rdkafka::Bindings.rd_kafka_destroy(native_client)
+  end
+
   subject { Rdkafka::Consumer::Message.new(native_message) }
 
   before do

data/spec/rdkafka/consumer_spec.rb

@@ -1,12 +1,16 @@
 require "spec_helper"
 require "ostruct"
+require 'securerandom'
 
 describe Rdkafka::Consumer do
   let(:config) { rdkafka_config }
   let(:consumer) { config.consumer }
   let(:producer) { config.producer }
 
-  describe "#subscripe, #unsubscribe and #subscription" do
+  after { consumer.close }
+  after { producer.close }
+
+  describe "#subscribe, #unsubscribe and #subscription" do
     it "should subscribe, unsubscribe and return the subscription" do
       expect(consumer.subscription).to be_empty
 
@@ -85,10 +89,9 @@ describe Rdkafka::Consumer do
         tpl.add_topic("consume_test_topic", (0..2))
         consumer.resume(tpl)
 
-        # 8. ensure that message is successfuly consumed
+        # 8. ensure that message is successfully consumed
         records = consumer.poll(timeout)
         expect(records).not_to be_nil
-        consumer.commit
       end
     end
 
@@ -205,8 +208,6 @@ describe Rdkafka::Consumer do
         expect(records&.payload).to eq "payload c"
         records = consumer.poll(timeout)
         expect(records).to be_nil
-
-        consumer.commit
       end
     end
   end
@@ -271,8 +272,18 @@ describe Rdkafka::Consumer do
   describe "#close" do
     it "should close a consumer" do
       consumer.subscribe("consume_test_topic")
+      100.times do |i|
+        report = producer.produce(
+          topic:     "consume_test_topic",
+          payload:   "payload #{i}",
+          key:       "key #{i}",
+          partition: 0
+        ).wait
+      end
       consumer.close
-      expect(consumer.poll(100)).to be_nil
+      expect {
+        consumer.poll(100)
+      }.to raise_error(Rdkafka::ClosedConsumerError, /poll/)
     end
   end
 
@@ -313,11 +324,11 @@ describe Rdkafka::Consumer do
       }.to raise_error TypeError
     end
 
-    context "with a commited consumer" do
+    context "with a committed consumer" do
       before :all do
-        # Make sure there are some message
-        producer = rdkafka_config.producer
+        # Make sure there are some messages.
         handles = []
+        producer = rdkafka_config.producer
         10.times do
           (0..2).each do |i|
             handles << producer.produce(
@@ -329,6 +340,7 @@ describe Rdkafka::Consumer do
           end
         end
         handles.each(&:wait)
+        producer.close
       end
 
       before do
@@ -389,20 +401,26 @@ describe Rdkafka::Consumer do
 
       describe "#store_offset" do
         before do
+          config = {}
           config[:'enable.auto.offset.store'] = false
           config[:'enable.auto.commit'] = false
-          consumer.subscribe("consume_test_topic")
-          wait_for_assignment(consumer)
+          @new_consumer = rdkafka_config(config).consumer
+          @new_consumer.subscribe("consume_test_topic")
+          wait_for_assignment(@new_consumer)
+        end
+
+        after do
+          @new_consumer.close
         end
 
         it "should store the offset for a message" do
-          consumer.store_offset(message)
-          consumer.commit
+          @new_consumer.store_offset(message)
+          @new_consumer.commit
 
           list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
             list.add_topic("consume_test_topic", [0, 1, 2])
           end
-          partitions = consumer.committed(list).to_h["consume_test_topic"]
+          partitions = @new_consumer.committed(list).to_h["consume_test_topic"]
           expect(partitions).not_to be_nil
           expect(partitions[message.partition].offset).to eq(message.offset + 1)
         end
@@ -410,7 +428,7 @@ describe Rdkafka::Consumer do
         it "should raise an error with invalid input" do
           allow(message).to receive(:partition).and_return(9999)
           expect {
-            consumer.store_offset(message)
+            @new_consumer.store_offset(message)
           }.to raise_error Rdkafka::RdkafkaError
         end
       end
@@ -554,12 +572,12 @@ describe Rdkafka::Consumer do
         payload:   "payload 1",
         key:       "key 1"
       ).wait
-
       consumer.subscribe("consume_test_topic")
-      message = consumer.poll(5000)
-      expect(message).to be_a Rdkafka::Consumer::Message
+      message = consumer.each {|m| break m}
 
-      # Message content is tested in producer spec
+      expect(message).to be_a Rdkafka::Consumer::Message
+      expect(message.payload).to eq('payload 1')
+      expect(message.key).to eq('key 1')
     end
 
     it "should raise an error when polling fails" do
@@ -655,7 +673,226 @@ describe Rdkafka::Consumer do
       # should break the each loop.
       consumer.each_with_index do |message, i|
         expect(message).to be_a Rdkafka::Consumer::Message
-        consumer.close if i == 10
+        break if i == 10
+      end
+      consumer.close
+    end
+  end
+
+  describe "#each_batch" do
+    let(:message_payload) { 'a' * 10 }
+
+    before do
+      @topic = SecureRandom.base64(10).tr('+=/', '')
+    end
+
+    after do
+      @topic = nil
+    end
+
+    def topic_name
+      @topic
+    end
+
+    def produce_n(n)
+      handles = []
+      n.times do |i|
+        handles << producer.produce(
+          topic:     topic_name,
+          payload:   Time.new.to_f.to_s,
+          key:       i.to_s,
+          partition: 0
+        )
+      end
+      handles.each(&:wait)
+    end
+
+    def new_message
+      instance_double("Rdkafka::Consumer::Message").tap do |message|
+        allow(message).to receive(:payload).and_return(message_payload)
+      end
+    end
+
+    it "retrieves messages produced into a topic" do
+      # This is the only each_batch test that actually produces real messages
+      # into a topic in the real kafka of the container.
+      #
+      # The other tests stub 'poll' which makes them faster and more reliable,
+      # but it makes sense to keep a single test with a fully integrated flow.
+      # This will help to catch breaking changes in the behavior of 'poll',
+      # libdrkafka, or Kafka.
+      #
+      # This is, in effect, an integration test and the subsequent specs are
+      # unit tests.
+      consumer.subscribe(topic_name)
+      produce_n 42
+      all_yields = []
+      consumer.each_batch(max_items: 10) do |batch|
+        all_yields << batch
+        break if all_yields.flatten.size >= 42
+      end
+      expect(all_yields.flatten.first).to be_a Rdkafka::Consumer::Message
+      expect(all_yields.flatten.size).to eq 42
+      expect(all_yields.size).to be > 4
+      expect(all_yields.flatten.map(&:key)).to eq (0..41).map { |x| x.to_s }
+    end
+
+    it "should batch poll results and yield arrays of messages" do
+      consumer.subscribe(topic_name)
+      all_yields = []
+      expect(consumer)
+        .to receive(:poll)
+        .exactly(10).times
+        .and_return(new_message)
+      consumer.each_batch(max_items: 10) do |batch|
+        all_yields << batch
+        break if all_yields.flatten.size >= 10
+      end
+      expect(all_yields.first).to be_instance_of(Array)
+      expect(all_yields.flatten.size).to eq 10
+      non_empty_yields = all_yields.reject { |batch| batch.empty? }
+      expect(non_empty_yields.size).to be < 10
+    end
+
+    it "should yield a partial batch if the timeout is hit with some messages" do
+      consumer.subscribe(topic_name)
+      poll_count = 0
+      expect(consumer)
+        .to receive(:poll)
+        .at_least(3).times do
+        poll_count = poll_count + 1
+        if poll_count > 2
+          sleep 0.1
+          nil
+        else
+          new_message
+        end
+      end
+      all_yields = []
+      consumer.each_batch(max_items: 10) do |batch|
+        all_yields << batch
+        break if all_yields.flatten.size >= 2
+      end
+      expect(all_yields.flatten.size).to eq 2
+    end
+
+    it "should yield [] if nothing is received before the timeout" do
+      consumer.subscribe(topic_name)
+      consumer.each_batch do |batch|
+        expect(batch).to eq([])
+        break
+      end
+    end
+
+    it "should yield batchs of max_items in size if messages are already fetched" do
+      yielded_batches = []
+      expect(consumer)
+        .to receive(:poll)
+        .with(anything)
+        .exactly(20).times
+        .and_return(new_message)
+
+      consumer.each_batch(max_items: 10, timeout_ms: 500) do |batch|
+        yielded_batches << batch
+        break if yielded_batches.flatten.size >= 20
+        break if yielded_batches.size >= 20 # so failure doesn't hang
+      end
+      expect(yielded_batches.size).to eq 2
+      expect(yielded_batches.map(&:size)).to eq 2.times.map { 10 }
+    end
+
+    it "should yield batchs as soon as bytes_threshold is hit" do
+      yielded_batches = []
+      expect(consumer)
+        .to receive(:poll)
+        .with(anything)
+        .exactly(20).times
+        .and_return(new_message)
+
+      consumer.each_batch(bytes_threshold: message_payload.size * 4, timeout_ms: 500) do |batch|
+        yielded_batches << batch
+        break if yielded_batches.flatten.size >= 20
+        break if yielded_batches.size >= 20 # so failure doesn't hang
+      end
+      expect(yielded_batches.size).to eq 5
+      expect(yielded_batches.map(&:size)).to eq 5.times.map { 4 }
+    end
+
+    context "error raised from poll and yield_on_error is true" do
+      it "should yield buffered exceptions on rebalance, then break" do
+        config = rdkafka_config({:"enable.auto.commit" => false,
+                                 :"enable.auto.offset.store" => false })
+        consumer = config.consumer
+        consumer.subscribe(topic_name)
+        loop_count = 0
+        batches_yielded = []
+        exceptions_yielded = []
+        each_batch_iterations = 0
+        poll_count = 0
+        expect(consumer)
+          .to receive(:poll)
+          .with(anything)
+          .exactly(3).times
+          .and_wrap_original do |method, *args|
+              poll_count = poll_count + 1
+              if poll_count == 3
+                raise Rdkafka::RdkafkaError.new(27,
+                    "partitions ... too ... heavy ... must ... rebalance")
+              else
+                new_message
+              end
+          end
+        expect {
+          consumer.each_batch(max_items: 30, yield_on_error: true) do |batch, pending_error|
+            batches_yielded << batch
+            exceptions_yielded << pending_error
+            each_batch_iterations = each_batch_iterations + 1
+          end
+        }.to raise_error(Rdkafka::RdkafkaError)
+        expect(poll_count).to eq 3
+        expect(each_batch_iterations).to eq 1
+        expect(batches_yielded.size).to eq 1
+        expect(batches_yielded.first.size).to eq 2
+        expect(exceptions_yielded.flatten.size).to eq 1
+        expect(exceptions_yielded.flatten.first).to be_instance_of(Rdkafka::RdkafkaError)
+      end
+    end
+
+    context "error raised from poll and yield_on_error is false" do
+      it "should yield buffered exceptions on rebalance, then break" do
+        config = rdkafka_config({:"enable.auto.commit" => false,
+                                 :"enable.auto.offset.store" => false })
+        consumer = config.consumer
+        consumer.subscribe(topic_name)
+        loop_count = 0
+        batches_yielded = []
+        exceptions_yielded = []
+        each_batch_iterations = 0
+        poll_count = 0
+        expect(consumer)
+          .to receive(:poll)
+          .with(anything)
+          .exactly(3).times
+          .and_wrap_original do |method, *args|
+              poll_count = poll_count + 1
+              if poll_count == 3
+                raise Rdkafka::RdkafkaError.new(27,
+                    "partitions ... too ... heavy ... must ... rebalance")
+              else
+                new_message
+              end
+          end
+        expect {
+          consumer.each_batch(max_items: 30, yield_on_error: false) do |batch, pending_error|
+            batches_yielded << batch
+            exceptions_yielded << pending_error
+            each_batch_iterations = each_batch_iterations + 1
+          end
+        }.to raise_error(Rdkafka::RdkafkaError)
+        expect(poll_count).to eq 3
+        expect(each_batch_iterations).to eq 0
+        expect(batches_yielded.size).to eq 0
+        expect(exceptions_yielded.size).to eq 0
       end
     end
   end
@@ -716,4 +953,34 @@ describe Rdkafka::Consumer do
       consumer.close
     end
   end
+
+  context "methods that should not be called after a consumer has been closed" do
+    before do
+      consumer.close
+    end
+
+    # Affected methods and a non-invalid set of parameters for the method
+    {
+        :subscribe               => [ nil ],
+        :unsubscribe             => nil,
+        :each_batch              => nil,
+        :pause                   => [ nil ],
+        :resume                  => [ nil ],
+        :subscription            => nil,
+        :assign                  => [ nil ],
+        :assignment              => nil,
+        :committed               => [],
+        :query_watermark_offsets => [ nil, nil ],
+    }.each do |method, args|
+      it "raises an exception if #{method} is called" do
+        expect {
+          if args.nil?
+            consumer.public_send(method)
+          else
+            consumer.public_send(method, *args)
+          end
+        }.to raise_exception(Rdkafka::ClosedConsumerError, /#{method.to_s}/)
+      end
+    end
+  end
 end