logstash-integration-kafka 10.0.1-java → 10.5.0-java

data/spec/integration/outputs/kafka_spec.rb

@@ -3,26 +3,29 @@
 require "logstash/devutils/rspec/spec_helper"
 require 'logstash/outputs/kafka'
 require 'json'
-require 'poseidon'
+require 'kafka'
 
 describe "outputs/kafka", :integration => true do
   let(:kafka_host) { 'localhost' }
   let(:kafka_port) { 9092 }
   let(:num_events) { 10 }
+
   let(:base_config) { {'client_id' => 'kafkaoutputspec'} }
-  let(:event) { LogStash::Event.new({'message' => '183.60.215.50 - - [11/Sep/2014:22:00:00 +0000] "GET /scripts/netcat-webserver HTTP/1.1" 200 182 "-" "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)"', '@timestamp' => LogStash::Timestamp.at(0) }) }
+  let(:message_content) do
+    '"GET /scripts/netcat-webserver HTTP/1.1" 200 182 "-" "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)"'
+  end
+  let(:event) do
+    LogStash::Event.new({ 'message' =>
+      '183.60.215.50 - - [11/Sep/2014:22:00:00 +0000] ' + message_content,
+      '@timestamp' => LogStash::Timestamp.at(0)
+    })
+  end
 
+  let(:kafka_client) { Kafka.new ["#{kafka_host}:#{kafka_port}"] }
 
   context 'when outputting messages serialized as String' do
     let(:test_topic) { 'logstash_integration_topic1' }
     let(:num_events) { 3 }
-    let(:consumer) do
-      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
-                                      test_topic, 0, :earliest_offset)
-    end
-    subject do
-      consumer.fetch
-    end
 
     before :each do
       config = base_config.merge({"topic_id" => test_topic})
@@ -30,8 +33,10 @@ describe "outputs/kafka", :integration => true do
     end
 
     it 'should have data integrity' do
-      expect(subject.size).to eq(num_events)
-      subject.each do |m|
+      messages = fetch_messages(test_topic)
+
+      expect(messages.size).to eq(num_events)
+      messages.each do |m|
         expect(m.value).to eq(event.to_s)
       end
     end
@@ -41,13 +46,6 @@ describe "outputs/kafka", :integration => true do
   context 'when outputting messages serialized as Byte Array' do
     let(:test_topic) { 'topic1b' }
     let(:num_events) { 3 }
-    let(:consumer) do
-      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
-                                      test_topic, 0, :earliest_offset)
-    end
-    subject do
-      consumer.fetch
-    end
 
     before :each do
       config = base_config.merge(
@@ -60,8 +58,10 @@ describe "outputs/kafka", :integration => true do
     end
 
     it 'should have data integrity' do
-      expect(subject.size).to eq(num_events)
-      subject.each do |m|
+      messages = fetch_messages(test_topic)
+
+      expect(messages.size).to eq(num_events)
+      messages.each do |m|
         expect(m.value).to eq(event.to_s)
       end
     end
@@ -71,14 +71,6 @@ describe "outputs/kafka", :integration => true do
   context 'when setting message_key' do
     let(:num_events) { 10 }
     let(:test_topic) { 'logstash_integration_topic2' }
-    let!(:consumer0) do
-      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
-                                      test_topic, 0, :earliest_offset)
-    end
-    let!(:consumer1) do
-      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
-                                      test_topic, 1, :earliest_offset)
-    end
 
     before :each do
       config = base_config.merge({"topic_id" => test_topic, "message_key" => "static_key"})
@@ -86,19 +78,14 @@ describe "outputs/kafka", :integration => true do
     end
 
     it 'should send all events to one partition' do
-      expect(consumer0.fetch.size == num_events || consumer1.fetch.size == num_events).to be true
+      data0 = fetch_messages(test_topic, partition: 0)
+      data1 = fetch_messages(test_topic, partition: 1)
+      expect(data0.size == num_events || data1.size == num_events).to be true
     end
   end
 
   context 'when using gzip compression' do
     let(:test_topic) { 'logstash_integration_gzip_topic' }
-    let!(:consumer) do
-      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
-                                      test_topic, 0, :earliest_offset)
-    end
-    subject do
-      consumer.fetch
-    end
 
     before :each do
       config = base_config.merge({"topic_id" => test_topic, "compression_type" => "gzip"})
@@ -106,8 +93,10 @@ describe "outputs/kafka", :integration => true do
     end
 
     it 'should have data integrity' do
-      expect(subject.size).to eq(num_events)
-      subject.each do |m|
+      messages = fetch_messages(test_topic)
+
+      expect(messages.size).to eq(num_events)
+      messages.each do |m|
         expect(m.value).to eq(event.to_s)
       end
     end
@@ -115,13 +104,6 @@ describe "outputs/kafka", :integration => true do
 
   context 'when using snappy compression' do
     let(:test_topic) { 'logstash_integration_snappy_topic' }
-    let!(:consumer) do
-      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
-                                      test_topic, 0, :earliest_offset)
-    end
-    subject do
-      consumer.fetch
-    end
 
     before :each do
       config = base_config.merge({"topic_id" => test_topic, "compression_type" => "snappy"})
@@ -129,8 +111,10 @@ describe "outputs/kafka", :integration => true do
     end
 
     it 'should have data integrity' do
-      expect(subject.size).to eq(num_events)
-      subject.each do |m|
+      messages = fetch_messages(test_topic)
+
+      expect(messages.size).to eq(num_events)
+      messages.each do |m|
         expect(m.value).to eq(event.to_s)
       end
     end
@@ -143,52 +127,85 @@ describe "outputs/kafka", :integration => true do
       config = base_config.merge({"topic_id" => test_topic, "compression_type" => "lz4"})
       load_kafka_data(config)
     end
+
+    # NOTE: depends on extlz4 gem which is using a C-extension
+    # it 'should have data integrity' do
+    #   messages = fetch_messages(test_topic)
+    #
+    #   expect(messages.size).to eq(num_events)
+    #   messages.each do |m|
+    #     expect(m.value).to eq(event.to_s)
+    #   end
+    # end
   end
 
   context 'when using multi partition topic' do
-    let(:num_events) { 10 }
+    let(:num_events) { 100 } # ~ more than (batch.size) 16,384 bytes
     let(:test_topic) { 'logstash_integration_topic3' }
-    let!(:consumer0) do
-      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
-                                      test_topic, 0, :earliest_offset)
-    end
-    let!(:consumer1) do
-      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
-                                      test_topic, 1, :earliest_offset)
+
+    before :each do
+      config = base_config.merge("topic_id" => test_topic, "partitioner" => 'org.apache.kafka.clients.producer.UniformStickyPartitioner')
+      load_kafka_data(config) do # let's have a bit more (diverse) dataset
+        num_events.times.collect do
+          LogStash::Event.new.tap do |e|
+            e.set('message', event.get('message').sub('183.60.215.50') { "#{rand(126)+1}.#{rand(126)+1}.#{rand(126)+1}.#{rand(126)+1}" })
+          end
+        end
+      end
     end
 
-    let!(:consumer2) do
-      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
-                                      test_topic, 2, :earliest_offset)
+    it 'should distribute events to all partitions' do
+      consumer0_records = fetch_messages(test_topic, partition: 0)
+      consumer1_records = fetch_messages(test_topic, partition: 1)
+      consumer2_records = fetch_messages(test_topic, partition: 2)
+
+      all_records = consumer0_records + consumer1_records + consumer2_records
+      expect(all_records.size).to eq(num_events * 2)
+      all_records.each do |m|
+        expect(m.value).to include message_content
+      end
+
+      expect(consumer0_records.size).to be > 1
+      expect(consumer1_records.size).to be > 1
+      expect(consumer2_records.size).to be > 1
     end
+  end
+
+  context 'setting partitioner' do
+    let(:test_topic) { 'logstash_integration_partitioner_topic' }
+    let(:partitioner) { nil }
 
     before :each do
-      config = base_config.merge({"topic_id" => test_topic})
+      @messages_offset = fetch_messages_from_all_partitions
+
+      config = base_config.merge("topic_id" => test_topic, 'partitioner' => partitioner)
       load_kafka_data(config)
     end
 
-    it 'should distribute events to all partition' do
-      consumer0_records = consumer0.fetch
-      consumer1_records = consumer1.fetch
-      consumer2_records = consumer2.fetch
-
-      expect(consumer0_records.size > 1 &&
-        consumer1_records.size > 1 &&
-          consumer2_records.size > 1).to be true
-
-      all_records = consumer0_records + consumer1_records + consumer2_records
-      expect(all_records.size).to eq(num_events)
-      all_records.each do |m|
-        expect(m.value).to eq(event.to_s)
+    [ 'default', 'round_robin', 'uniform_sticky' ].each do |partitioner|
+      describe partitioner do
+        let(:partitioner) { partitioner }
+        it 'loads data' do
+          expect(fetch_messages_from_all_partitions - @messages_offset).to eql num_events
+        end
       end
     end
+
+    def fetch_messages_from_all_partitions
+      3.times.map { |i| fetch_messages(test_topic, partition: i).size }.sum
+    end
   end
 
   def load_kafka_data(config)
     kafka = LogStash::Outputs::Kafka.new(config)
     kafka.register
     kafka.multi_receive(num_events.times.collect { event })
+    kafka.multi_receive(Array(yield)) if block_given?
     kafka.close
   end
 
+  def fetch_messages(topic, partition: 0, offset: :earliest)
+    kafka_client.fetch_messages(topic: topic, partition: partition, offset: offset)
+  end
+
 end

data/spec/unit/inputs/kafka_spec.rb

@@ -34,6 +34,68 @@ describe LogStash::Inputs::Kafka do
   subject { LogStash::Inputs::Kafka.new(config) }
 
   it "should register" do
-    expect {subject.register}.to_not raise_error
+    expect { subject.register }.to_not raise_error
+  end
+
+  context 'with client_rack' do
+    let(:config) { super.merge('client_rack' => 'EU-R1') }
+
+    it "sets broker rack parameter" do
+      expect(org.apache.kafka.clients.consumer.KafkaConsumer).
+          to receive(:new).with(hash_including('client.rack' => 'EU-R1')).
+              and_return kafka_client = double('kafka-consumer')
+
+      expect( subject.send(:create_consumer, 'sample_client-0') ).to be kafka_client
+    end
+  end
+
+  context 'string integer config' do
+    let(:config) { super.merge('session_timeout_ms' => '25000', 'max_poll_interval_ms' => '345000') }
+
+    it "sets integer values" do
+      expect(org.apache.kafka.clients.consumer.KafkaConsumer).
+          to receive(:new).with(hash_including('session.timeout.ms' => '25000', 'max.poll.interval.ms' => '345000')).
+              and_return kafka_client = double('kafka-consumer')
+
+      expect( subject.send(:create_consumer, 'sample_client-1') ).to be kafka_client
+    end
+  end
+
+  context 'integer config' do
+    let(:config) { super.merge('session_timeout_ms' => 25200, 'max_poll_interval_ms' => 123_000) }
+
+    it "sets integer values" do
+      expect(org.apache.kafka.clients.consumer.KafkaConsumer).
+          to receive(:new).with(hash_including('session.timeout.ms' => '25200', 'max.poll.interval.ms' => '123000')).
+              and_return kafka_client = double('kafka-consumer')
+
+      expect( subject.send(:create_consumer, 'sample_client-2') ).to be kafka_client
+    end
+  end
+
+  context 'string boolean config' do
+    let(:config) { super.merge('enable_auto_commit' => 'false', 'check_crcs' => 'true') }
+
+    it "sets parameters" do
+      expect(org.apache.kafka.clients.consumer.KafkaConsumer).
+          to receive(:new).with(hash_including('enable.auto.commit' => 'false', 'check.crcs' => 'true')).
+              and_return kafka_client = double('kafka-consumer')
+
+      expect( subject.send(:create_consumer, 'sample_client-3') ).to be kafka_client
+      expect( subject.enable_auto_commit ).to be false
+    end
+  end
+
+  context 'boolean config' do
+    let(:config) { super.merge('enable_auto_commit' => true, 'check_crcs' => false) }
+
+    it "sets parameters" do
+      expect(org.apache.kafka.clients.consumer.KafkaConsumer).
+          to receive(:new).with(hash_including('enable.auto.commit' => 'true', 'check.crcs' => 'false')).
+              and_return kafka_client = double('kafka-consumer')
+
+      expect( subject.send(:create_consumer, 'sample_client-4') ).to be kafka_client
+      expect( subject.enable_auto_commit ).to be true
+    end
   end
 end

data/spec/unit/outputs/kafka_spec.rb

@@ -56,14 +56,15 @@ describe "outputs/kafka" do
     end
   end
 
-  context "when KafkaProducer#send() raises an exception" do
+  context "when KafkaProducer#send() raises a retriable exception" do
     let(:failcount) { (rand * 10).to_i }
     let(:sendcount) { failcount + 1 }
 
     let(:exception_classes) { [
       org.apache.kafka.common.errors.TimeoutException,
+      org.apache.kafka.common.errors.DisconnectException,
+      org.apache.kafka.common.errors.CoordinatorNotAvailableException,
       org.apache.kafka.common.errors.InterruptException,
-      org.apache.kafka.common.errors.SerializationException
     ] }
 
     before do
@@ -88,6 +89,37 @@ describe "outputs/kafka" do
     end
   end
 
+  context "when KafkaProducer#send() raises a non-retriable exception" do
+    let(:failcount) { (rand * 10).to_i }
+
+    let(:exception_classes) { [
+        org.apache.kafka.common.errors.SerializationException,
+        org.apache.kafka.common.errors.RecordTooLargeException,
+        org.apache.kafka.common.errors.InvalidTopicException
+    ] }
+
+    before do
+      count = 0
+      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
+        .exactly(1).times
+        .and_wrap_original do |m, *args|
+        if count < failcount # fail 'failcount' times in a row.
+          count += 1
+          # Pick an exception at random
+          raise exception_classes.shuffle.first.new("injected exception for testing")
+        else
+          m.call(*args) # call original
+        end
+      end
+    end
+
+    it "should not retry" do
+      kafka = LogStash::Outputs::Kafka.new(simple_kafka_config)
+      kafka.register
+      kafka.multi_receive([event])
+    end
+  end
+
   context "when a send fails" do
     context "and the default retries behavior is used" do
       # Fail this many times and then finally succeed.
@@ -97,7 +129,7 @@ describe "outputs/kafka" do
       let(:sendcount) { failcount + 1 }
 
       it "should retry until successful" do
-        count = 0;
+        count = 0
 
         expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
               .exactly(sendcount).times
@@ -107,7 +139,7 @@ describe "outputs/kafka" do
             # inject some failures.
 
             # Return a custom Future that will raise an exception to simulate a Kafka send() problem.
-            future = java.util.concurrent.FutureTask.new { raise "Failed" }
+            future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
             future.run
             future
           else
@@ -129,7 +161,7 @@ describe "outputs/kafka" do
                                                                                        .once
                                                                                        .and_wrap_original do |m, *args|
           # Always fail.
-          future = java.util.concurrent.FutureTask.new { raise "Failed" }
+          future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
           future.run
           future
         end
@@ -143,7 +175,7 @@ describe "outputs/kafka" do
                                                                                        .once
                                                                                        .and_wrap_original do |m, *args|
           # Always fail.
-          future = java.util.concurrent.FutureTask.new { raise "Failed" }
+          future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
           future.run
           future
         end
@@ -164,7 +196,7 @@ describe "outputs/kafka" do
               .at_most(max_sends).times
               .and_wrap_original do |m, *args|
           # Always fail.
-          future = java.util.concurrent.FutureTask.new { raise "Failed" }
+          future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
           future.run
           future
         end
@@ -175,10 +207,10 @@ describe "outputs/kafka" do
 
       it 'should only sleep retries number of times' do
         expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
-                                                                                       .at_most(max_sends)
+                                                                                       .at_most(max_sends).times
                                                                                        .and_wrap_original do |m, *args|
           # Always fail.
-          future = java.util.concurrent.FutureTask.new { raise "Failed" }
+          future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
           future.run
           future
         end
@@ -189,4 +221,25 @@ describe "outputs/kafka" do
       end
     end
   end
+
+  context 'when ssl endpoint identification disabled' do
+
+    let(:config) do
+      simple_kafka_config.merge('ssl_endpoint_identification_algorithm' => '', 'security_protocol' => 'SSL')
+    end
+
+    subject { LogStash::Outputs::Kafka.new(config) }
+
+    it 'does not configure truststore' do
+      expect(org.apache.kafka.clients.producer.KafkaProducer).
+          to receive(:new).with(hash_excluding('ssl.truststore.location' => anything))
+      subject.register
+    end
+
+    it 'sets empty ssl.endpoint.identification.algorithm' do
+      expect(org.apache.kafka.clients.producer.KafkaProducer).
+          to receive(:new).with(hash_including('ssl.endpoint.identification.algorithm' => ''))
+      subject.register
+    end
+  end
 end