logstash-integration-kafka 10.0.0-java → 10.4.0-java

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '10.0.0'
+  s.version         = '10.4.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+
@@ -35,7 +35,7 @@ Gem::Specification.new do |s|
       "integration_plugins" => "logstash-input-kafka,logstash-output-kafka"
   }
 
-  s.add_development_dependency 'jar-dependencies', '~> 0.3.12'
+  s.add_development_dependency 'jar-dependencies', '>= 0.3.12'
 
   s.platform = RUBY_PLATFORM
 
@@ -49,6 +49,6 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'rspec-wait'
-  s.add_development_dependency 'poseidon'
+  s.add_development_dependency 'ruby-kafka'
   s.add_development_dependency 'snappy'
 end

data/spec/integration/inputs/kafka_spec.rb

@@ -1,7 +1,6 @@
 # encoding: utf-8
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/inputs/kafka"
-require "digest"
 require "rspec/wait"
 
 # Please run kafka_test_setup.sh prior to executing this integration test.
@@ -12,159 +11,129 @@ describe "inputs/kafka", :integration => true do
   let(:group_id_3) {rand(36**8).to_s(36)}
   let(:group_id_4) {rand(36**8).to_s(36)}
   let(:group_id_5) {rand(36**8).to_s(36)}
-  let(:plain_config) { { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'} }
-  let(:multi_consumer_config) { plain_config.merge({"group_id" => group_id_4, "client_id" => "spec", "consumer_threads" => 3}) }
-  let(:snappy_config) { { 'topics' => ['logstash_integration_topic_snappy'], 'codec' => 'plain', 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'} }
-  let(:lz4_config) { { 'topics' => ['logstash_integration_topic_lz4'], 'codec' => 'plain', 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'} }
-  let(:pattern_config) { { 'topics_pattern' => 'logstash_integration_topic_.*', 'group_id' => group_id_2, 'codec' => 'plain', 'auto_offset_reset' => 'earliest'} }
-  let(:decorate_config) { { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_3, 'auto_offset_reset' => 'earliest', 'decorate_events' => true} }
-  let(:manual_commit_config) { { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_5, 'auto_offset_reset' => 'earliest', 'enable_auto_commit' => 'false'} }
+  let(:group_id_6) {rand(36**8).to_s(36)}
+  let(:plain_config) do
+    { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_1,
+      'auto_offset_reset' => 'earliest' }
+  end
+  let(:multi_consumer_config) do
+    plain_config.merge({"group_id" => group_id_4, "client_id" => "spec", "consumer_threads" => 3})
+  end
+  let(:snappy_config) do
+    { 'topics' => ['logstash_integration_topic_snappy'], 'codec' => 'plain', 'group_id' => group_id_1,
+      'auto_offset_reset' => 'earliest' }
+  end
+  let(:lz4_config) do
+    { 'topics' => ['logstash_integration_topic_lz4'], 'codec' => 'plain', 'group_id' => group_id_1,
+      'auto_offset_reset' => 'earliest' }
+  end
+  let(:pattern_config) do
+    { 'topics_pattern' => 'logstash_integration_topic_.*', 'group_id' => group_id_2, 'codec' => 'plain',
+      'auto_offset_reset' => 'earliest' }
+  end
+  let(:decorate_config) do
+    { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_3,
+      'auto_offset_reset' => 'earliest', 'decorate_events' => true }
+  end
+  let(:manual_commit_config) do
+    { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_5,
+      'auto_offset_reset' => 'earliest', 'enable_auto_commit' => 'false' }
+  end
   let(:timeout_seconds) { 30 }
   let(:num_events) { 103 }
 
   describe "#kafka-topics" do
-    def thread_it(kafka_input, queue)
-      Thread.new do
-        begin
-          kafka_input.run(queue)
-        end
-      end
-    end
 
     it "should consume all messages from plain 3-partition topic" do
-      kafka_input = LogStash::Inputs::Kafka.new(plain_config)
-      queue = Queue.new
-      t = thread_it(kafka_input, queue)
-      begin
-        t.run
-        wait(timeout_seconds).for {queue.length}.to eq(num_events)
-        expect(queue.length).to eq(num_events)
-      ensure
-        t.kill
-        t.join(30_000)
-      end
+      queue = consume_messages(plain_config, timeout: timeout_seconds, event_count: num_events)
+      expect(queue.length).to eq(num_events)
     end
 
     it "should consume all messages from snappy 3-partition topic" do
-      kafka_input = LogStash::Inputs::Kafka.new(snappy_config)
-      queue = Queue.new
-      t = thread_it(kafka_input, queue)
-      begin
-        t.run
-        wait(timeout_seconds).for {queue.length}.to eq(num_events)
-        expect(queue.length).to eq(num_events)
-      ensure
-        t.kill
-        t.join(30_000)
-      end
+      queue = consume_messages(snappy_config, timeout: timeout_seconds, event_count: num_events)
+      expect(queue.length).to eq(num_events)
     end
 
     it "should consume all messages from lz4 3-partition topic" do
-      kafka_input = LogStash::Inputs::Kafka.new(lz4_config)
-      queue = Queue.new
-      t = thread_it(kafka_input, queue)
-      begin
-        t.run
-        wait(timeout_seconds).for {queue.length}.to eq(num_events)
-        expect(queue.length).to eq(num_events)
-      ensure
-        t.kill
-        t.join(30_000)
-      end
+      queue = consume_messages(lz4_config, timeout: timeout_seconds, event_count: num_events)
+      expect(queue.length).to eq(num_events)
     end
 
     it "should consumer all messages with multiple consumers" do
-      kafka_input = LogStash::Inputs::Kafka.new(multi_consumer_config)
-      queue = Queue.new
-      t = thread_it(kafka_input, queue)
-      begin
-        t.run
-        wait(timeout_seconds).for {queue.length}.to eq(num_events)
+      consume_messages(multi_consumer_config, timeout: timeout_seconds, event_count: num_events) do |queue, kafka_input|
         expect(queue.length).to eq(num_events)
         kafka_input.kafka_consumers.each_with_index do |consumer, i|
           expect(consumer.metrics.keys.first.tags["client-id"]).to eq("spec-#{i}")
         end
-      ensure
-        t.kill
-        t.join(30_000)
       end
     end
   end
 
-  describe "#kafka-topics-pattern" do
-    def thread_it(kafka_input, queue)
-      Thread.new do
-        begin
-          kafka_input.run(queue)
-        end
-      end
-    end
-
+  context "#kafka-topics-pattern" do
     it "should consume all messages from all 3 topics" do
-      kafka_input = LogStash::Inputs::Kafka.new(pattern_config)
-      queue = Queue.new
-      t = thread_it(kafka_input, queue)
-      begin
-        t.run
-        wait(timeout_seconds).for {queue.length}.to eq(3*num_events)
-        expect(queue.length).to eq(3*num_events)
-      ensure
-        t.kill
-        t.join(30_000)
-      end
+      total_events = num_events * 3
+      queue = consume_messages(pattern_config, timeout: timeout_seconds, event_count: total_events)
+      expect(queue.length).to eq(total_events)
     end
   end
 
-  describe "#kafka-decorate" do
-    def thread_it(kafka_input, queue)
-      Thread.new do
-        begin
-          kafka_input.run(queue)
-        end
-      end
-    end
-
+  context "#kafka-decorate" do
     it "should show the right topic and group name in decorated kafka section" do
       start = LogStash::Timestamp.now.time.to_i
-      kafka_input = LogStash::Inputs::Kafka.new(decorate_config)
-      queue = Queue.new
-      t = thread_it(kafka_input, queue)
-      begin
-        t.run
-        wait(timeout_seconds).for {queue.length}.to eq(num_events)
+      consume_messages(decorate_config, timeout: timeout_seconds, event_count: num_events) do |queue, _|
         expect(queue.length).to eq(num_events)
         event = queue.shift
         expect(event.get("[@metadata][kafka][topic]")).to eq("logstash_integration_topic_plain")
         expect(event.get("[@metadata][kafka][consumer_group]")).to eq(group_id_3)
         expect(event.get("[@metadata][kafka][timestamp]")).to be >= start
-      ensure
-        t.kill
-        t.join(30_000)
       end
     end
   end
 
-  describe "#kafka-offset-commit" do
-    def thread_it(kafka_input, queue)
-      Thread.new do
-        begin
-          kafka_input.run(queue)
+  context "#kafka-offset-commit" do
+    it "should manually commit offsets" do
+      queue = consume_messages(manual_commit_config, timeout: timeout_seconds, event_count: num_events)
+      expect(queue.length).to eq(num_events)
+    end
+  end
+
+  context 'setting partition_assignment_strategy' do
+    let(:test_topic) { 'logstash_integration_partitioner_topic' }
+    let(:consumer_config) do
+      plain_config.merge(
+          "topics" => [test_topic],
+          'group_id' => group_id_6,
+          "client_id" => "partition_assignment_strategy-spec",
+          "consumer_threads" => 2,
+          "partition_assignment_strategy" => partition_assignment_strategy
+      )
+    end
+    let(:partition_assignment_strategy) { nil }
+
+    # NOTE: just verify setting works, as its a bit cumbersome to do in a unit spec
+    [ 'range', 'round_robin', 'sticky', 'org.apache.kafka.clients.consumer.CooperativeStickyAssignor' ].each do |partition_assignment_strategy|
+      describe partition_assignment_strategy do
+        let(:partition_assignment_strategy) { partition_assignment_strategy }
+        it 'consumes data' do
+          consume_messages(consumer_config, timeout: false, event_count: 0)
         end
       end
     end
+  end
 
-    it "should manually commit offsets" do
-      kafka_input = LogStash::Inputs::Kafka.new(manual_commit_config)
-      queue = Queue.new
-      t = thread_it(kafka_input, queue)
-      begin
-        t.run
-        wait(timeout_seconds).for {queue.length}.to eq(num_events)
-        expect(queue.length).to eq(num_events)
-      ensure
-        t.kill
-        t.join(30_000)
-      end
+  private
+
+  def consume_messages(config, queue: Queue.new, timeout:, event_count:)
+    kafka_input = LogStash::Inputs::Kafka.new(config)
+    t = Thread.new { kafka_input.run(queue) }
+    begin
+      t.run
+      wait(timeout).for { queue.length }.to eq(event_count) unless timeout.eql?(false)
+      block_given? ? yield(queue, kafka_input) : queue
+    ensure
+      t.kill
+      t.join(30_000)
     end
   end
+
 end

data/spec/integration/outputs/kafka_spec.rb

@@ -3,26 +3,29 @@
 require "logstash/devutils/rspec/spec_helper"
 require 'logstash/outputs/kafka'
 require 'json'
-require 'poseidon'
+require 'kafka'
 
 describe "outputs/kafka", :integration => true do
   let(:kafka_host) { 'localhost' }
   let(:kafka_port) { 9092 }
   let(:num_events) { 10 }
+
   let(:base_config) { {'client_id' => 'kafkaoutputspec'} }
-  let(:event) { LogStash::Event.new({'message' => '183.60.215.50 - - [11/Sep/2014:22:00:00 +0000] "GET /scripts/netcat-webserver HTTP/1.1" 200 182 "-" "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)"', '@timestamp' => LogStash::Timestamp.at(0) }) }
+  let(:message_content) do
+    '"GET /scripts/netcat-webserver HTTP/1.1" 200 182 "-" "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)"'
+  end
+  let(:event) do
+    LogStash::Event.new({ 'message' =>
+      '183.60.215.50 - - [11/Sep/2014:22:00:00 +0000] ' + message_content,
+      '@timestamp' => LogStash::Timestamp.at(0)
+    })
+  end
 
+  let(:kafka_client) { Kafka.new ["#{kafka_host}:#{kafka_port}"] }
 
   context 'when outputting messages serialized as String' do
     let(:test_topic) { 'logstash_integration_topic1' }
     let(:num_events) { 3 }
-    let(:consumer) do
-      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
-                                      test_topic, 0, :earliest_offset)
-    end
-    subject do
-      consumer.fetch
-    end
 
     before :each do
       config = base_config.merge({"topic_id" => test_topic})
@@ -30,8 +33,10 @@ describe "outputs/kafka", :integration => true do
     end
 
     it 'should have data integrity' do
-      expect(subject.size).to eq(num_events)
-      subject.each do |m|
+      messages = fetch_messages(test_topic)
+
+      expect(messages.size).to eq(num_events)
+      messages.each do |m|
         expect(m.value).to eq(event.to_s)
       end
     end
@@ -41,13 +46,6 @@ describe "outputs/kafka", :integration => true do
   context 'when outputting messages serialized as Byte Array' do
     let(:test_topic) { 'topic1b' }
     let(:num_events) { 3 }
-    let(:consumer) do
-      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
-                                      test_topic, 0, :earliest_offset)
-    end
-    subject do
-      consumer.fetch
-    end
 
     before :each do
       config = base_config.merge(
@@ -60,8 +58,10 @@ describe "outputs/kafka", :integration => true do
     end
 
     it 'should have data integrity' do
-      expect(subject.size).to eq(num_events)
-      subject.each do |m|
+      messages = fetch_messages(test_topic)
+
+      expect(messages.size).to eq(num_events)
+      messages.each do |m|
         expect(m.value).to eq(event.to_s)
       end
     end
@@ -71,14 +71,6 @@ describe "outputs/kafka", :integration => true do
   context 'when setting message_key' do
     let(:num_events) { 10 }
     let(:test_topic) { 'logstash_integration_topic2' }
-    let!(:consumer0) do
-      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
-                                      test_topic, 0, :earliest_offset)
-    end
-    let!(:consumer1) do
-      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
-                                      test_topic, 1, :earliest_offset)
-    end
 
     before :each do
       config = base_config.merge({"topic_id" => test_topic, "message_key" => "static_key"})
@@ -86,19 +78,14 @@ describe "outputs/kafka", :integration => true do
     end
 
     it 'should send all events to one partition' do
-      expect(consumer0.fetch.size == num_events || consumer1.fetch.size == num_events).to be true
+      data0 = fetch_messages(test_topic, partition: 0)
+      data1 = fetch_messages(test_topic, partition: 1)
+      expect(data0.size == num_events || data1.size == num_events).to be true
     end
   end
 
   context 'when using gzip compression' do
     let(:test_topic) { 'logstash_integration_gzip_topic' }
-    let!(:consumer) do
-      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
-                                      test_topic, 0, :earliest_offset)
-    end
-    subject do
-      consumer.fetch
-    end
 
     before :each do
       config = base_config.merge({"topic_id" => test_topic, "compression_type" => "gzip"})
@@ -106,8 +93,10 @@ describe "outputs/kafka", :integration => true do
     end
 
     it 'should have data integrity' do
-      expect(subject.size).to eq(num_events)
-      subject.each do |m|
+      messages = fetch_messages(test_topic)
+
+      expect(messages.size).to eq(num_events)
+      messages.each do |m|
         expect(m.value).to eq(event.to_s)
       end
     end
@@ -115,13 +104,6 @@ describe "outputs/kafka", :integration => true do
 
   context 'when using snappy compression' do
     let(:test_topic) { 'logstash_integration_snappy_topic' }
-    let!(:consumer) do
-      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
-                                      test_topic, 0, :earliest_offset)
-    end
-    subject do
-      consumer.fetch
-    end
 
     before :each do
       config = base_config.merge({"topic_id" => test_topic, "compression_type" => "snappy"})
@@ -129,8 +111,10 @@ describe "outputs/kafka", :integration => true do
     end
 
     it 'should have data integrity' do
-      expect(subject.size).to eq(num_events)
-      subject.each do |m|
+      messages = fetch_messages(test_topic)
+
+      expect(messages.size).to eq(num_events)
+      messages.each do |m|
         expect(m.value).to eq(event.to_s)
       end
     end
@@ -143,52 +127,85 @@ describe "outputs/kafka", :integration => true do
       config = base_config.merge({"topic_id" => test_topic, "compression_type" => "lz4"})
       load_kafka_data(config)
     end
+
+    # NOTE: depends on extlz4 gem which is using a C-extension
+    # it 'should have data integrity' do
+    #   messages = fetch_messages(test_topic)
+    #
+    #   expect(messages.size).to eq(num_events)
+    #   messages.each do |m|
+    #     expect(m.value).to eq(event.to_s)
+    #   end
+    # end
   end
 
   context 'when using multi partition topic' do
-    let(:num_events) { 10 }
+    let(:num_events) { 100 } # ~ more than (batch.size) 16,384 bytes
     let(:test_topic) { 'logstash_integration_topic3' }
-    let!(:consumer0) do
-      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
-                                      test_topic, 0, :earliest_offset)
-    end
-    let!(:consumer1) do
-      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
-                                      test_topic, 1, :earliest_offset)
+
+    before :each do
+      config = base_config.merge("topic_id" => test_topic, "partitioner" => 'org.apache.kafka.clients.producer.UniformStickyPartitioner')
+      load_kafka_data(config) do # let's have a bit more (diverse) dataset
+        num_events.times.collect do
+          LogStash::Event.new.tap do |e|
+            e.set('message', event.get('message').sub('183.60.215.50') { "#{rand(126)+1}.#{rand(126)+1}.#{rand(126)+1}.#{rand(126)+1}" })
+          end
+        end
+      end
     end
 
-    let!(:consumer2) do
-      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
-                                      test_topic, 2, :earliest_offset)
+    it 'should distribute events to all partitions' do
+      consumer0_records = fetch_messages(test_topic, partition: 0)
+      consumer1_records = fetch_messages(test_topic, partition: 1)
+      consumer2_records = fetch_messages(test_topic, partition: 2)
+
+      all_records = consumer0_records + consumer1_records + consumer2_records
+      expect(all_records.size).to eq(num_events * 2)
+      all_records.each do |m|
+        expect(m.value).to include message_content
+      end
+
+      expect(consumer0_records.size).to be > 1
+      expect(consumer1_records.size).to be > 1
+      expect(consumer2_records.size).to be > 1
     end
+  end
+
+  context 'setting partitioner' do
+    let(:test_topic) { 'logstash_integration_partitioner_topic' }
+    let(:partitioner) { nil }
 
     before :each do
-      config = base_config.merge({"topic_id" => test_topic})
+      @messages_offset = fetch_messages_from_all_partitions
+
+      config = base_config.merge("topic_id" => test_topic, 'partitioner' => partitioner)
       load_kafka_data(config)
     end
 
-    it 'should distribute events to all partition' do
-      consumer0_records = consumer0.fetch
-      consumer1_records = consumer1.fetch
-      consumer2_records = consumer2.fetch
-
-      expect(consumer0_records.size > 1 &&
-        consumer1_records.size > 1 &&
-          consumer2_records.size > 1).to be true
-
-      all_records = consumer0_records + consumer1_records + consumer2_records
-      expect(all_records.size).to eq(num_events)
-      all_records.each do |m|
-        expect(m.value).to eq(event.to_s)
+    [ 'default', 'round_robin', 'uniform_sticky' ].each do |partitioner|
+      describe partitioner do
+        let(:partitioner) { partitioner }
+        it 'loads data' do
+          expect(fetch_messages_from_all_partitions - @messages_offset).to eql num_events
+        end
       end
     end
+
+    def fetch_messages_from_all_partitions
+      3.times.map { |i| fetch_messages(test_topic, partition: i).size }.sum
+    end
   end
 
   def load_kafka_data(config)
     kafka = LogStash::Outputs::Kafka.new(config)
     kafka.register
     kafka.multi_receive(num_events.times.collect { event })
+    kafka.multi_receive(Array(yield)) if block_given?
     kafka.close
   end
 
+  def fetch_messages(topic, partition: 0, offset: :earliest)
+    kafka_client.fetch_messages(topic: topic, partition: partition, offset: offset)
+  end
+
 end