logstash-input-kafka 5.1.4 → 5.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 55e8135ee758f6822896fe75496f99d1f2702782
-  data.tar.gz: b6cda942dc0f5d129879efb1ad5a568b3353aeae
+  metadata.gz: 69bf688e6da796bc836086907a8511aefa3a4a8d
+  data.tar.gz: 41980f6d7375bf9daae7fba455cd089a83bde427
 SHA512:
-  metadata.gz: ad0a3fd16cdaf547da5b8adb6cd8890474b60bdc3644233ace4fb4c8016b5b6fbbf3c187d1ac4beafaec6301e8313cb95685d5609d61092d4cea525305723d0b
-  data.tar.gz: 6e2347e9f29175dd565bc44389e7d6124a358f4be1209ccafcd39b7f87dac1fff5add835931ce32c9e096dab09b36abb47451e0f8569453ab10f5138e28eb1ea
+  metadata.gz: 6ea54547d2e8beaefd0a8a67ac7a29fd08a35c27710a8ee48da70102c0a5e20dcaeda23c1993483334cf0728730ec763ed74cc1110bf0e193b1def045b4f63a0
+  data.tar.gz: a6783693e93d7fde277c062929ded62298a7bf37b0f82fd3419051f5421426e72a7a5977e989e169b8cd0630a1939055ad7c1fa3869c2e6e8b7da69f24187861

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 5.1.6
+  - fix: Client ID is no longer reused across multiple Kafka consumer instances
+
+## 5.1.5
+  - Fix a bug where consumer was not correctly setup when `SASL_SSL` option was specified. 
+
 ## 5.1.4
   - Fix log4j bootstrapping for 2.4
 

data/lib/logstash/inputs/kafka.rb

@@ -209,7 +209,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
 
   public
   def run(logstash_queue)
-    @runner_consumers = consumer_threads.times.map { || create_consumer }
+    @runner_consumers = consumer_threads.times.map { |i| create_consumer("#{client_id}-#{i}") }
     @runner_threads = @runner_consumers.map { |consumer| thread_runner(logstash_queue, consumer) }
     @runner_threads.each { |t| t.join }
   end # def run
@@ -219,6 +219,11 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
     @runner_consumers.each { |c| c.wakeup }
   end
 
+  public
+  def kafka_consumers
+    @runner_consumers
+  end
+
   private
   def thread_runner(logstash_queue, consumer)
     Thread.new do
@@ -255,7 +260,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   end
 
   private
-  def create_consumer
+  def create_consumer(client_id)
     begin
       props = java.util.Properties.new
       kafka = org.apache.kafka.clients.consumer.ConsumerConfig
@@ -292,8 +297,8 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
       elsif security_protocol == "SASL_PLAINTEXT"
         set_sasl_config(props)
       elsif security_protocol == "SASL_SSL"
-        set_trustore_keystore_config
-        set_sasl_config
+        set_trustore_keystore_config(props)
+        set_sasl_config(props)
       end
 
       org.apache.kafka.clients.consumer.KafkaConsumer.new(props)

data/logstash-input-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-kafka'
-  s.version         = '5.1.4'
+  s.version         = '5.1.6'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = 'This input will read events from a Kafka topic. It uses the high level consumer API provided by Kafka to read messages from the broker'
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/integration/inputs/kafka_spec.rb

@@ -10,7 +10,9 @@ describe "inputs/kafka", :integration => true do
   let(:group_id_1) {rand(36**8).to_s(36)}
   let(:group_id_2) {rand(36**8).to_s(36)}
   let(:group_id_3) {rand(36**8).to_s(36)}
+  let(:group_id_4) {rand(36**8).to_s(36)}
   let(:plain_config) { { 'topics' => ['logstash_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'} }
+  let(:multi_consumer_config) { plain_config.merge({"group_id" => group_id_4, "client_id" => "spec", "consumer_threads" => 3}) }
   let(:snappy_config) { { 'topics' => ['logstash_topic_snappy'], 'codec' => 'plain', 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'} }
   let(:lz4_config) { { 'topics' => ['logstash_topic_lz4'], 'codec' => 'plain', 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'} }
   let(:pattern_config) { { 'topics_pattern' => 'logstash_topic_.*', 'group_id' => group_id_2, 'codec' => 'plain', 'auto_offset_reset' => 'earliest'} }  
@@ -53,11 +55,21 @@ describe "inputs/kafka", :integration => true do
       wait(timeout_seconds).for { queue.length }.to eq(num_events)
       expect(queue.length).to eq(num_events)
     end
-    
+
+    it "should consumer all messages with multiple consumers" do
+      kafka_input = LogStash::Inputs::Kafka.new(multi_consumer_config)
+      queue = Array.new
+      t = thread_it(kafka_input, queue)
+      t.run
+      wait(timeout_seconds).for { queue.length }.to eq(num_events)
+      expect(queue.length).to eq(num_events)
+      kafka_input.kafka_consumers.each_with_index do |consumer, i|
+        expect(consumer.metrics.keys.first.tags["client-id"]).to eq("spec-#{i}")
+      end
+    end
   end
 
   describe "#kafka-topics-pattern" do
-    
     def thread_it(kafka_input, queue)
       Thread.new do
         begin
@@ -65,7 +77,7 @@ describe "inputs/kafka", :integration => true do
         end
       end
     end
-      
+
     it "should consume all messages from all 3 topics" do
       kafka_input = LogStash::Inputs::Kafka.new(pattern_config)
       queue = Array.new
@@ -73,7 +85,7 @@ describe "inputs/kafka", :integration => true do
       t.run
       wait(timeout_seconds).for { queue.length }.to eq(3*num_events)
       expect(queue.length).to eq(3*num_events)
-    end    
+    end
   end
 
   describe "#kafka-decorate" do
@@ -84,7 +96,7 @@ describe "inputs/kafka", :integration => true do
         end
       end
     end
-      
+
     it "should show the right topic and group name in decorated kafka section" do
       kafka_input = LogStash::Inputs::Kafka.new(decorate_config)
       queue = Queue.new

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-kafka
 version: !ruby/object:Gem::Version
-  version: 5.1.4
+  version: 5.1.6
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-01-10 00:00:00.000000000 Z
+date: 2017-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement