logstash-input-kafka 6.2.4 → 6.2.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 600b946684433e0a63bddc646760c4039c2a4433
-  data.tar.gz: 70b05de15c5fb8de308150417785b920a6d3aa3b
+  metadata.gz: a81f9cd12e2e3d4b62a79fdc783dbbec3c23914a
+  data.tar.gz: 7b3a63847c2345526d913dd33df657965faffd86
 SHA512:
-  metadata.gz: 91a770392e5d18c9386ef03e107edd17f2ced7c4fbaf6dae84e18bfffe6bbae581403153426271c08544412290a332e5c08c426be117513a614ee6d2e96963d9
-  data.tar.gz: 0bdd87a8228d485680810e95fc896db151144f4ae57be137cfc12cfaed8e62c6a6885e14618f24ef5ac78f3ebe5817d9064abf1281f1a987c91b72606f4dd8fd
+  metadata.gz: c0161ad18db27f054b2997df14383d9c7308e5dbaa2e0fb869ab84758ff0232a349a71c56cc28a7c89294e7b103053aaba61566719b90bba83f737c17a9832ef
+  data.tar.gz: 7e089f9dafe7e99833b30b20317356dd4eadc0743ebce480b33aa61dca599d4063e606a0db904bfec21f8f67e51721d6d6fda0af117a31b3fa273ed9fea6c9da

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 6.2.6
+  - fix: Client ID is no longer reused across multiple Kafka consumer instances
+
+## 6.2.5
+  - Fix a bug where consumer was not correctly setup when `SASL_SSL` option was specified. 
+
 ## 6.2.4
   - Make error reporting more clear when connection fails 
 

data/lib/logstash/inputs/kafka.rb

@@ -220,7 +220,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
 
   public
   def run(logstash_queue)
-    @runner_consumers = consumer_threads.times.map { || create_consumer }
+    @runner_consumers = consumer_threads.times.map { |i| create_consumer("#{client_id}-#{i}") }
     @runner_threads = @runner_consumers.map { |consumer| thread_runner(logstash_queue, consumer) }
     @runner_threads.each { |t| t.join }
   end # def run
@@ -230,6 +230,11 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
     @runner_consumers.each { |c| c.wakeup }
   end
 
+  public
+  def kafka_consumers
+    @runner_consumers
+  end
+
   private
   def thread_runner(logstash_queue, consumer)
     Thread.new do
@@ -266,7 +271,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   end
 
   private
-  def create_consumer
+  def create_consumer(client_id)
     begin
       props = java.util.Properties.new
       kafka = org.apache.kafka.clients.consumer.ConsumerConfig
@@ -305,8 +310,8 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
       elsif security_protocol == "SASL_PLAINTEXT"
         set_sasl_config(props)
       elsif security_protocol == "SASL_SSL"
-        set_trustore_keystore_config
-        set_sasl_config
+        set_trustore_keystore_config(props)
+        set_sasl_config(props)
       end
 
       org.apache.kafka.clients.consumer.KafkaConsumer.new(props)

data/logstash-input-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-kafka'
-  s.version         = '6.2.4'
+  s.version         = '6.2.6'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = 'This input will read events from a Kafka topic. It uses the high level consumer API provided by Kafka to read messages from the broker'
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/integration/inputs/kafka_spec.rb

@@ -10,7 +10,9 @@ describe "inputs/kafka", :integration => true do
   let(:group_id_1) {rand(36**8).to_s(36)}
   let(:group_id_2) {rand(36**8).to_s(36)}
   let(:group_id_3) {rand(36**8).to_s(36)}
+  let(:group_id_4) {rand(36**8).to_s(36)}
   let(:plain_config) { { 'topics' => ['logstash_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'} }
+  let(:multi_consumer_config) { plain_config.merge({"group_id" => group_id_4, "client_id" => "spec", "consumer_threads" => 3}) }
   let(:snappy_config) { { 'topics' => ['logstash_topic_snappy'], 'codec' => 'plain', 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'} }
   let(:lz4_config) { { 'topics' => ['logstash_topic_lz4'], 'codec' => 'plain', 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'} }
   let(:pattern_config) { { 'topics_pattern' => 'logstash_topic_.*', 'group_id' => group_id_2, 'codec' => 'plain', 'auto_offset_reset' => 'earliest'} }  
@@ -53,11 +55,21 @@ describe "inputs/kafka", :integration => true do
       wait(timeout_seconds).for { queue.length }.to eq(num_events)
       expect(queue.length).to eq(num_events)
     end
-    
+
+    it "should consumer all messages with multiple consumers" do
+      kafka_input = LogStash::Inputs::Kafka.new(multi_consumer_config)
+      queue = Array.new
+      t = thread_it(kafka_input, queue)
+      t.run
+      wait(timeout_seconds).for { queue.length }.to eq(num_events)
+      expect(queue.length).to eq(num_events)
+      kafka_input.kafka_consumers.each_with_index do |consumer, i|
+        expect(consumer.metrics.keys.first.tags["client-id"]).to eq("spec-#{i}")
+      end
+    end
   end
 
   describe "#kafka-topics-pattern" do
-    
     def thread_it(kafka_input, queue)
       Thread.new do
         begin
@@ -65,7 +77,7 @@ describe "inputs/kafka", :integration => true do
         end
       end
     end
-      
+
     it "should consume all messages from all 3 topics" do
       kafka_input = LogStash::Inputs::Kafka.new(pattern_config)
       queue = Array.new
@@ -73,7 +85,7 @@ describe "inputs/kafka", :integration => true do
       t.run
       wait(timeout_seconds).for { queue.length }.to eq(3*num_events)
       expect(queue.length).to eq(3*num_events)
-    end    
+    end
   end
 
   describe "#kafka-decorate" do
@@ -84,7 +96,7 @@ describe "inputs/kafka", :integration => true do
         end
       end
     end
-      
+
     it "should show the right topic and group name in decorated kafka section" do
       kafka_input = LogStash::Inputs::Kafka.new(decorate_config)
       queue = Queue.new

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-kafka
 version: !ruby/object:Gem::Version
-  version: 6.2.4
+  version: 6.2.6
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-01-10 00:00:00.000000000 Z
+date: 2017-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement