rdkafka 0.8.0 → 0.11.0

data/spec/rdkafka/bindings_spec.rb

@@ -25,39 +25,39 @@ describe Rdkafka::Bindings do
   end
 
   describe "log callback" do
-    let(:log) { StringIO.new }
+    let(:log_queue) { Rdkafka::Config.log_queue }
     before do
-      Rdkafka::Config.logger = Logger.new(log)
+      allow(log_queue).to receive(:<<)
     end
 
     it "should log fatal messages" do
       Rdkafka::Bindings::LogCallback.call(nil, 0, nil, "log line")
-      expect(log.string).to include "FATAL -- : rdkafka: log line"
+      expect(log_queue).to have_received(:<<).with([Logger::FATAL, "rdkafka: log line"])
     end
 
     it "should log error messages" do
       Rdkafka::Bindings::LogCallback.call(nil, 3, nil, "log line")
-      expect(log.string).to include "ERROR -- : rdkafka: log line"
+      expect(log_queue).to have_received(:<<).with([Logger::ERROR, "rdkafka: log line"])
     end
 
     it "should log warning messages" do
       Rdkafka::Bindings::LogCallback.call(nil, 4, nil, "log line")
-      expect(log.string).to include "WARN -- : rdkafka: log line"
+      expect(log_queue).to have_received(:<<).with([Logger::WARN, "rdkafka: log line"])
     end
 
     it "should log info messages" do
       Rdkafka::Bindings::LogCallback.call(nil, 5, nil, "log line")
-      expect(log.string).to include "INFO -- : rdkafka: log line"
+      expect(log_queue).to have_received(:<<).with([Logger::INFO, "rdkafka: log line"])
     end
 
     it "should log debug messages" do
       Rdkafka::Bindings::LogCallback.call(nil, 7, nil, "log line")
-      expect(log.string).to include "DEBUG -- : rdkafka: log line"
+      expect(log_queue).to have_received(:<<).with([Logger::DEBUG, "rdkafka: log line"])
     end
 
     it "should log unknown messages" do
       Rdkafka::Bindings::LogCallback.call(nil, 100, nil, "log line")
-      expect(log.string).to include "ANY -- : rdkafka: log line"
+      expect(log_queue).to have_received(:<<).with([Logger::UNKNOWN, "rdkafka: log line"])
     end
   end
 
@@ -100,4 +100,28 @@ describe Rdkafka::Bindings do
       end
     end
   end
+
+  describe "error callback" do
+    context "without an error callback" do
+      it "should do nothing" do
+        expect {
+          Rdkafka::Bindings::ErrorCallback.call(nil, 1, "error", nil)
+        }.not_to raise_error
+      end
+    end
+
+    context "with an error callback" do
+      before do
+        Rdkafka::Config.error_callback = lambda do |error|
+          $received_error = error
+        end
+      end
+
+      it "should call the error callback with an Rdkafka::Error" do
+        Rdkafka::Bindings::ErrorCallback.call(nil, 8, "Broker not available", nil)
+        expect($received_error.code).to eq(:broker_not_available)
+        expect($received_error.broker_message).to eq("Broker not available")
+      end
+    end
+  end
 end

data/spec/rdkafka/callbacks_spec.rb

@@ -0,0 +1,20 @@
+require "spec_helper"
+
+describe Rdkafka::Callbacks do
+
+  # The code in the call back functions is 100% covered by other specs.  Due to
+  # the large number of collaborators, and the fact that FFI does not play
+  # nicely with doubles, it was very difficult to construct tests that were
+  # not over-mocked.
+
+  # For debugging purposes, if you suspect that you are running into trouble in
+  # one of the callback functions, it may be helpful to surround the inner body
+  # of the method with something like:
+  #
+  #   begin
+  #     <method body>
+  #   rescue => ex; puts ex.inspect; puts ex.backtrace; end;
+  #
+  # This will output to STDOUT any exceptions that are being raised in the callback.
+
+end

data/spec/rdkafka/config_spec.rb

@@ -18,25 +18,83 @@ describe Rdkafka::Config do
         Rdkafka::Config.logger = nil
       }.to raise_error(Rdkafka::Config::NoLoggerError)
     end
+
+    it "supports logging queue" do
+      log = StringIO.new
+      Rdkafka::Config.logger = Logger.new(log)
+
+      Rdkafka::Config.log_queue << [Logger::FATAL, "I love testing"]
+      20.times do
+        break if log.string != ""
+        sleep 0.05
+      end
+
+      expect(log.string).to include "FATAL -- : I love testing"
+    end
   end
 
   context "statistics callback" do
-    it "should set the callback" do
-      expect {
-        Rdkafka::Config.statistics_callback = lambda do |stats|
-          puts stats
+    context "with a proc/lambda" do
+      it "should set the callback" do
+        expect {
+          Rdkafka::Config.statistics_callback = lambda do |stats|
+            puts stats
+          end
+        }.not_to raise_error
+        expect(Rdkafka::Config.statistics_callback).to respond_to :call
+      end
+    end
+
+    context "with a callable object" do
+      it "should set the callback" do
+        callback = Class.new do
+          def call(stats); end
         end
-      }.not_to raise_error
-      expect(Rdkafka::Config.statistics_callback).to be_a Proc
+        expect {
+          Rdkafka::Config.statistics_callback = callback.new
+        }.not_to raise_error
+        expect(Rdkafka::Config.statistics_callback).to respond_to :call
+      end
     end
 
-    it "should not accept a callback that's not a proc" do
+    it "should not accept a callback that's not callable" do
       expect {
         Rdkafka::Config.statistics_callback = 'a string'
       }.to raise_error(TypeError)
     end
   end
 
+  context "error callback" do
+    context "with a proc/lambda" do
+      it "should set the callback" do
+        expect {
+          Rdkafka::Config.error_callback = lambda do |error|
+            puts error
+          end
+        }.not_to raise_error
+        expect(Rdkafka::Config.error_callback).to respond_to :call
+      end
+    end
+
+    context "with a callable object" do
+      it "should set the callback" do
+        callback = Class.new do
+          def call(stats); end
+        end
+        expect {
+          Rdkafka::Config.error_callback = callback.new
+        }.not_to raise_error
+        expect(Rdkafka::Config.error_callback).to respond_to :call
+      end
+    end
+
+    it "should not accept a callback that's not callable" do
+      expect {
+        Rdkafka::Config.error_callback = 'a string'
+      }.to raise_error(TypeError)
+    end
+  end
+
   context "configuration" do
     it "should store configuration" do
       config = Rdkafka::Config.new
@@ -50,7 +108,7 @@ describe Rdkafka::Config do
     end
 
     it "should create a consumer with valid config" do
-      consumer = rdkafka_config.consumer
+      consumer = rdkafka_consumer_config.consumer
       expect(consumer).to be_a Rdkafka::Consumer
       consumer.close
     end
@@ -78,7 +136,7 @@ describe Rdkafka::Config do
     end
 
     it "should create a producer with valid config" do
-      producer = rdkafka_config.producer
+      producer = rdkafka_consumer_config.producer
       expect(producer).to be_a Rdkafka::Producer
       producer.close
     end
@@ -90,6 +148,17 @@ describe Rdkafka::Config do
       }.to raise_error(Rdkafka::Config::ConfigError, "No such configuration property: \"invalid.key\"")
     end
 
+    it "should allow configuring zstd compression" do
+      config = Rdkafka::Config.new('compression.codec' => 'zstd')
+      begin
+        expect(config.producer).to be_a Rdkafka::Producer
+        config.producer.close
+      rescue Rdkafka::Config::ConfigError => ex
+        pending "Zstd compression not supported on this machine"
+        raise ex
+      end
+    end
+
     it "should raise an error when client creation fails for a consumer" do
       config = Rdkafka::Config.new(
         "security.protocol" => "SSL",

data/spec/rdkafka/consumer_spec.rb

@@ -1,10 +1,10 @@
 require "spec_helper"
 require "ostruct"
+require 'securerandom'
 
 describe Rdkafka::Consumer do
-  let(:config) { rdkafka_config }
-  let(:consumer) { config.consumer }
-  let(:producer) { config.producer }
+  let(:consumer) { rdkafka_consumer_config.consumer }
+  let(:producer) { rdkafka_producer_config.producer }
 
   after { consumer.close }
   after { producer.close }
@@ -271,8 +271,18 @@ describe Rdkafka::Consumer do
   describe "#close" do
     it "should close a consumer" do
       consumer.subscribe("consume_test_topic")
+      100.times do |i|
+        report = producer.produce(
+          topic:     "consume_test_topic",
+          payload:   "payload #{i}",
+          key:       "key #{i}",
+          partition: 0
+        ).wait
+      end
       consumer.close
-      expect(consumer.poll(100)).to be_nil
+      expect {
+        consumer.poll(100)
+      }.to raise_error(Rdkafka::ClosedConsumerError, /poll/)
     end
   end
 
@@ -317,7 +327,7 @@ describe Rdkafka::Consumer do
       before :all do
         # Make sure there are some messages.
         handles = []
-        producer = rdkafka_config.producer
+        producer = rdkafka_producer_config.producer
         10.times do
           (0..2).each do |i|
             handles << producer.produce(
@@ -393,7 +403,7 @@ describe Rdkafka::Consumer do
           config = {}
           config[:'enable.auto.offset.store'] = false
           config[:'enable.auto.commit'] = false
-          @new_consumer = rdkafka_config(config).consumer
+          @new_consumer = rdkafka_consumer_config(config).consumer
           @new_consumer.subscribe("consume_test_topic")
           wait_for_assignment(@new_consumer)
         end
@@ -448,13 +458,13 @@ describe Rdkafka::Consumer do
   end
 
   describe "#lag" do
-    let(:config) { rdkafka_config(:"enable.partition.eof" => true) }
+    let(:consumer) { rdkafka_consumer_config(:"enable.partition.eof" => true).consumer }
 
     it "should calculate the consumer lag" do
       # Make sure there's a message in every partition and
       # wait for the message to make sure everything is committed.
       (0..2).each do |i|
-        report = producer.produce(
+        producer.produce(
           topic:     "consume_test_topic",
           key:       "key lag #{i}",
           partition: i
@@ -497,7 +507,7 @@ describe Rdkafka::Consumer do
 
       # Produce message on every topic again
       (0..2).each do |i|
-        report = producer.produce(
+        producer.produce(
           topic:     "consume_test_topic",
           key:       "key lag #{i}",
           partition: i
@@ -662,57 +672,301 @@ describe Rdkafka::Consumer do
       # should break the each loop.
       consumer.each_with_index do |message, i|
         expect(message).to be_a Rdkafka::Consumer::Message
-        consumer.close if i == 10
+        break if i == 10
       end
+      consumer.close
     end
   end
 
-  describe "a rebalance listener" do
-    it "should get notifications" do
-      listener = Struct.new(:queue) do
-        def on_partitions_assigned(consumer, list)
-          collect(:assign, list)
-        end
+  describe "#each_batch" do
+    let(:message_payload) { 'a' * 10 }
 
-        def on_partitions_revoked(consumer, list)
-          collect(:revoke, list)
-        end
+    before do
+      @topic = SecureRandom.base64(10).tr('+=/', '')
+    end
 
-        def collect(name, list)
-          partitions = list.to_h.map { |key, values| [key, values.map(&:partition)] }.flatten
-          queue << ([name] + partitions)
-        end
-      end.new([])
+    after do
+      @topic = nil
+    end
 
-      notify_listener(listener)
+    def topic_name
+      @topic
+    end
 
-      expect(listener.queue).to eq([
-        [:assign, "consume_test_topic", 0, 1, 2],
-        [:revoke, "consume_test_topic", 0, 1, 2]
-      ])
+    def produce_n(n)
+      handles = []
+      n.times do |i|
+        handles << producer.produce(
+          topic:     topic_name,
+          payload:   Time.new.to_f.to_s,
+          key:       i.to_s,
+          partition: 0
+        )
+      end
+      handles.each(&:wait)
     end
 
-    it 'should handle callback exceptions' do
-      listener = Struct.new(:queue) do
-        def on_partitions_assigned(consumer, list)
-          queue << :assigned
-          raise 'boom'
-        end
+    def new_message
+      instance_double("Rdkafka::Consumer::Message").tap do |message|
+        allow(message).to receive(:payload).and_return(message_payload)
+      end
+    end
 
-        def on_partitions_revoked(consumer, list)
-          queue << :revoked
-          raise 'boom'
+    it "retrieves messages produced into a topic" do
+      # This is the only each_batch test that actually produces real messages
+      # into a topic in the real kafka of the container.
+      #
+      # The other tests stub 'poll' which makes them faster and more reliable,
+      # but it makes sense to keep a single test with a fully integrated flow.
+      # This will help to catch breaking changes in the behavior of 'poll',
+      # libdrkafka, or Kafka.
+      #
+      # This is, in effect, an integration test and the subsequent specs are
+      # unit tests.
+      create_topic_handle = rdkafka_config.admin.create_topic(topic_name, 1, 1)
+      create_topic_handle.wait(max_wait_timeout: 15.0)
+      consumer.subscribe(topic_name)
+      produce_n 42
+      all_yields = []
+      consumer.each_batch(max_items: 10) do |batch|
+        all_yields << batch
+        break if all_yields.flatten.size >= 42
+      end
+      expect(all_yields.flatten.first).to be_a Rdkafka::Consumer::Message
+      expect(all_yields.flatten.size).to eq 42
+      expect(all_yields.size).to be > 4
+      expect(all_yields.flatten.map(&:key)).to eq (0..41).map { |x| x.to_s }
+    end
+
+    it "should batch poll results and yield arrays of messages" do
+      consumer.subscribe(topic_name)
+      all_yields = []
+      expect(consumer)
+        .to receive(:poll)
+        .exactly(10).times
+        .and_return(new_message)
+      consumer.each_batch(max_items: 10) do |batch|
+        all_yields << batch
+        break if all_yields.flatten.size >= 10
+      end
+      expect(all_yields.first).to be_instance_of(Array)
+      expect(all_yields.flatten.size).to eq 10
+      non_empty_yields = all_yields.reject { |batch| batch.empty? }
+      expect(non_empty_yields.size).to be < 10
+    end
+
+    it "should yield a partial batch if the timeout is hit with some messages" do
+      consumer.subscribe(topic_name)
+      poll_count = 0
+      expect(consumer)
+        .to receive(:poll)
+        .at_least(3).times do
+        poll_count = poll_count + 1
+        if poll_count > 2
+          sleep 0.1
+          nil
+        else
+          new_message
         end
-      end.new([])
+      end
+      all_yields = []
+      consumer.each_batch(max_items: 10) do |batch|
+        all_yields << batch
+        break if all_yields.flatten.size >= 2
+      end
+      expect(all_yields.flatten.size).to eq 2
+    end
+
+    it "should yield [] if nothing is received before the timeout" do
+      create_topic_handle = rdkafka_config.admin.create_topic(topic_name, 1, 1)
+      create_topic_handle.wait(max_wait_timeout: 15.0)
+      consumer.subscribe(topic_name)
+      consumer.each_batch do |batch|
+        expect(batch).to eq([])
+        break
+      end
+    end
 
-      notify_listener(listener)
+    it "should yield batchs of max_items in size if messages are already fetched" do
+      yielded_batches = []
+      expect(consumer)
+        .to receive(:poll)
+        .with(anything)
+        .exactly(20).times
+        .and_return(new_message)
 
-      expect(listener.queue).to eq([:assigned, :revoked])
+      consumer.each_batch(max_items: 10, timeout_ms: 500) do |batch|
+        yielded_batches << batch
+        break if yielded_batches.flatten.size >= 20
+        break if yielded_batches.size >= 20 # so failure doesn't hang
+      end
+      expect(yielded_batches.size).to eq 2
+      expect(yielded_batches.map(&:size)).to eq 2.times.map { 10 }
+    end
+
+    it "should yield batchs as soon as bytes_threshold is hit" do
+      yielded_batches = []
+      expect(consumer)
+        .to receive(:poll)
+        .with(anything)
+        .exactly(20).times
+        .and_return(new_message)
+
+      consumer.each_batch(bytes_threshold: message_payload.size * 4, timeout_ms: 500) do |batch|
+        yielded_batches << batch
+        break if yielded_batches.flatten.size >= 20
+        break if yielded_batches.size >= 20 # so failure doesn't hang
+      end
+      expect(yielded_batches.size).to eq 5
+      expect(yielded_batches.map(&:size)).to eq 5.times.map { 4 }
+    end
+
+    context "error raised from poll and yield_on_error is true" do
+      it "should yield buffered exceptions on rebalance, then break" do
+        config = rdkafka_consumer_config(
+          {
+            :"enable.auto.commit" => false,
+            :"enable.auto.offset.store" => false
+          }
+        )
+        consumer = config.consumer
+        consumer.subscribe(topic_name)
+        loop_count = 0
+        batches_yielded = []
+        exceptions_yielded = []
+        each_batch_iterations = 0
+        poll_count = 0
+        expect(consumer)
+          .to receive(:poll)
+          .with(anything)
+          .exactly(3).times
+          .and_wrap_original do |method, *args|
+              poll_count = poll_count + 1
+              if poll_count == 3
+                raise Rdkafka::RdkafkaError.new(27,
+                    "partitions ... too ... heavy ... must ... rebalance")
+              else
+                new_message
+              end
+          end
+        expect {
+          consumer.each_batch(max_items: 30, yield_on_error: true) do |batch, pending_error|
+            batches_yielded << batch
+            exceptions_yielded << pending_error
+            each_batch_iterations = each_batch_iterations + 1
+          end
+        }.to raise_error(Rdkafka::RdkafkaError)
+        expect(poll_count).to eq 3
+        expect(each_batch_iterations).to eq 1
+        expect(batches_yielded.size).to eq 1
+        expect(batches_yielded.first.size).to eq 2
+        expect(exceptions_yielded.flatten.size).to eq 1
+        expect(exceptions_yielded.flatten.first).to be_instance_of(Rdkafka::RdkafkaError)
+      end
+    end
+
+    context "error raised from poll and yield_on_error is false" do
+      it "should yield buffered exceptions on rebalance, then break" do
+        config = rdkafka_consumer_config(
+          {
+            :"enable.auto.commit" => false,
+            :"enable.auto.offset.store" => false
+          }
+        )
+        consumer = config.consumer
+        consumer.subscribe(topic_name)
+        loop_count = 0
+        batches_yielded = []
+        exceptions_yielded = []
+        each_batch_iterations = 0
+        poll_count = 0
+        expect(consumer)
+          .to receive(:poll)
+          .with(anything)
+          .exactly(3).times
+          .and_wrap_original do |method, *args|
+              poll_count = poll_count + 1
+              if poll_count == 3
+                raise Rdkafka::RdkafkaError.new(27,
+                    "partitions ... too ... heavy ... must ... rebalance")
+              else
+                new_message
+              end
+          end
+        expect {
+          consumer.each_batch(max_items: 30, yield_on_error: false) do |batch, pending_error|
+            batches_yielded << batch
+            exceptions_yielded << pending_error
+            each_batch_iterations = each_batch_iterations + 1
+          end
+        }.to raise_error(Rdkafka::RdkafkaError)
+        expect(poll_count).to eq 3
+        expect(each_batch_iterations).to eq 0
+        expect(batches_yielded.size).to eq 0
+        expect(exceptions_yielded.size).to eq 0
+      end
+    end
+  end
+
+  describe "a rebalance listener" do
+    let(:consumer) do
+      config = rdkafka_consumer_config
+      config.consumer_rebalance_listener = listener
+      config.consumer
+    end
+
+    context "with a working listener" do
+      let(:listener) do
+        Struct.new(:queue) do
+          def on_partitions_assigned(consumer, list)
+            collect(:assign, list)
+          end
+
+          def on_partitions_revoked(consumer, list)
+            collect(:revoke, list)
+          end
+
+          def collect(name, list)
+            partitions = list.to_h.map { |key, values| [key, values.map(&:partition)] }.flatten
+            queue << ([name] + partitions)
+          end
+        end.new([])
+      end
+
+      it "should get notifications" do
+        notify_listener(listener)
+
+        expect(listener.queue).to eq([
+          [:assign, "consume_test_topic", 0, 1, 2],
+          [:revoke, "consume_test_topic", 0, 1, 2]
+        ])
+      end
+    end
+
+    context "with a broken listener" do
+      let(:listener) do
+        Struct.new(:queue) do
+          def on_partitions_assigned(consumer, list)
+            queue << :assigned
+            raise 'boom'
+          end
+
+          def on_partitions_revoked(consumer, list)
+            queue << :revoked
+            raise 'boom'
+          end
+        end.new([])
+      end
+
+      it 'should handle callback exceptions' do
+        notify_listener(listener)
+
+        expect(listener.queue).to eq([:assigned, :revoked])
+      end
     end
 
     def notify_listener(listener)
       # 1. subscribe and poll
-      config.consumer_rebalance_listener = listener
       consumer.subscribe("consume_test_topic")
       wait_for_assignment(consumer)
       consumer.poll(100)
@@ -723,4 +977,34 @@ describe Rdkafka::Consumer do
       consumer.close
     end
   end
+
+  context "methods that should not be called after a consumer has been closed" do
+    before do
+      consumer.close
+    end
+
+    # Affected methods and a non-invalid set of parameters for the method
+    {
+        :subscribe               => [ nil ],
+        :unsubscribe             => nil,
+        :each_batch              => nil,
+        :pause                   => [ nil ],
+        :resume                  => [ nil ],
+        :subscription            => nil,
+        :assign                  => [ nil ],
+        :assignment              => nil,
+        :committed               => [],
+        :query_watermark_offsets => [ nil, nil ],
+    }.each do |method, args|
+      it "raises an exception if #{method} is called" do
+        expect {
+          if args.nil?
+            consumer.public_send(method)
+          else
+            consumer.public_send(method, *args)
+          end
+        }.to raise_exception(Rdkafka::ClosedConsumerError, /#{method.to_s}/)
+      end
+    end
+  end
 end

data/spec/rdkafka/error_spec.rb

@@ -11,6 +11,10 @@ describe Rdkafka::RdkafkaError do
     expect(Rdkafka::RdkafkaError.new(10, "message prefix").message_prefix).to eq "message prefix"
   end
 
+  it "should create an error with a broker message" do
+    expect(Rdkafka::RdkafkaError.new(10, broker_message: "broker message").broker_message).to eq "broker message"
+  end
+
   describe "#code" do
     it "should handle an invalid response" do
       expect(Rdkafka::RdkafkaError.new(933975).code).to eq :err_933975?