logstash-input-kafka 4.1.1 → 4.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a42988d8cbbaab2c7c76ab5dc51e73fafc8007d5
-  data.tar.gz: 531ac5d2c767ce9ad6df4edc08cffa69fcefc6b2
+  metadata.gz: 392af7e1646f486c6f81b5ed7dd2206335be8287
+  data.tar.gz: 75f322acc9c088a79a6e87c51349237d6bff91ed
 SHA512:
-  metadata.gz: 2a73db8d884a53a6958d664282948a0e2ff8c330b5b62915738c469e94f55c7d77540e3b2a1f49aabb39b3cc3ccc036503b9ea0d888c111739db30021158eac7
-  data.tar.gz: 524190a231cd0fa95356b80f38eed6d3bef1229c9bfdd8b7118e12ac753a7044338fbc8abe903671aefe2514fd8f8e55ecae200ed30893370f5e1d144d35545f
+  metadata.gz: 2541883711e5a2fdb1acf216fdf1f25a2af1ed266c84293a869805f3e7e418c2d6a162e5f57cece82dbadf63256d8edf8271be4b6e489e93381f7f8ca8ad3a93
+  data.tar.gz: 6b2fc56fbafe56be7c8a891f44b1c83dea8759e49856b8fc94da6e1378edcf69a7bd7e1b1fddbfaf2e8a333674d97a165fe9b151efd08b1168081cc35aef1be0

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 4.2.0
+  - Add topic pattern config which allows consuming from multiple topics
+  - Restored event decorators which adds metadata.
+
 ## 4.1.1
   - fix: vendor aliasing issue when publishing
 

data/lib/logstash/inputs/kafka.rb

@@ -114,8 +114,11 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   config :session_timeout_ms, :validate => :string, :default => "30000"
   # Java Class used to deserialize the record's value
   config :value_deserializer_class, :validate => :string, :default => "org.apache.kafka.common.serialization.StringDeserializer"
-  # A list of topics to subscribe to.
-  config :topics, :validate => :array, :required => true
+  # A list of topics to subscribe to, defaults to ["logstash"].
+  config :topics, :validate => :array, :default => ["logstash"]
+  # A topic regex pattern to subscribe to. 
+  # The topics configuration will be ignored when using this configuration.
+  config :topics_pattern, :validate => :string
   # Time kafka consumer will wait to receive new messages from topics
   config :poll_timeout_ms, :validate => :number, :default => 100
   # Enable SSL/TLS secured communication to Kafka broker. Note that secure communication 
@@ -129,6 +132,14 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   config :ssl_keystore_location, :validate => :path
   # If client authentication is required, this setting stores the keystore password
   config :ssl_keystore_password, :validate => :password
+  # Option to add Kafka metadata like topic, message size to the event.
+  # This will add a field named `kafka` to the logstash event containing the following attributes:
+  #   `topic`: The topic this message is associated with
+  #   `consumer_group`: The consumer group used to read in this event
+  #   `partition`: The partition this message is associated with
+  #   `offset`: The offset from the partition this message is associated with
+  #   `key`: A ByteBuffer containing the message key
+  config :decorate_events, :validate => :boolean, :default => false
 
   
   public
@@ -156,12 +167,25 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   def thread_runner(logstash_queue, consumer)
     Thread.new do
       begin
-        consumer.subscribe(topics);
+        unless @topics_pattern.nil?
+          nooplistener = org.apache.kafka.clients.consumer.internals.NoOpConsumerRebalanceListener.new
+          pattern = java.util.regex.Pattern.compile(@topics_pattern)
+          consumer.subscribe(pattern, nooplistener)
+        else
+          consumer.subscribe(topics);
+        end
         while !stop?
           records = consumer.poll(poll_timeout_ms);
           for record in records do
             @codec.decode(record.value.to_s) do |event|
               decorate(event)
+              if @decorate_events
+                event.set("[kafka][topic]", record.topic)
+                event.set("[kafka][consumer_group]", @group_id)
+                event.set("[kafka][partition]", record.partition)
+                event.set("[kafka][offset]", record.offset)
+                event.set("[kafka][key]", record.key)
+              end
               logstash_queue << event
             end
           end

data/logstash-input-kafka.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-kafka'
-  s.version         = '4.1.1'
+  s.version         = '4.2.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = 'This input will read events from a Kafka topic. It uses the high level consumer API provided by Kafka to read messages from the broker'
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/integration/inputs/kafka_spec.rb

@@ -2,59 +2,99 @@
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/inputs/kafka"
 require "digest"
+require "rspec/wait"
 
-describe "input/kafka", :integration => true do
-  let(:partition3_config) { { 'topics' => ['topic3'], 'codec' => 'plain', 'auto_offset_reset' => 'earliest'} }
-  let(:snappy_config) { { 'topics' => ['snappy_topic'], 'codec' => 'plain', 'auto_offset_reset' => 'earliest'} }
-  let(:lz4_config) { { 'topics' => ['lz4_topic'], 'codec' => 'plain', 'auto_offset_reset' => 'earliest'} }
-  
-  let(:tries) { 60 }
+# Please run kafka_test_setup.sh prior to executing this integration test.
+describe "inputs/kafka", :integration => true do
+  # Group ids to make sure that the consumers get all the logs.
+  let(:group_id_1) {rand(36**8).to_s(36)}
+  let(:group_id_2) {rand(36**8).to_s(36)}
+  let(:group_id_3) {rand(36**8).to_s(36)}
+  let(:plain_config) { { 'topics' => ['logstash_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'} }
+  let(:snappy_config) { { 'topics' => ['logstash_topic_snappy'], 'codec' => 'plain', 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'} }
+  let(:lz4_config) { { 'topics' => ['logstash_topic_lz4'], 'codec' => 'plain', 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'} }
+  let(:pattern_config) { { 'topics_pattern' => 'logstash_topic_.*', 'group_id' => group_id_2, 'codec' => 'plain', 'auto_offset_reset' => 'earliest'} }  
+  let(:decorate_config) { { 'topics' => ['logstash_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_3, 'auto_offset_reset' => 'earliest', 'decorate_events' => true} }
+  let(:timeout_seconds) { 120 }
   let(:num_events) { 103 }
-  
-  def thread_it(kafka_input, queue)
-    Thread.new do
-      begin
-        kafka_input.run(queue)
+
+  describe "#kafka-topics" do
+    def thread_it(kafka_input, queue)
+      Thread.new do
+        begin
+          kafka_input.run(queue)
+        end
       end
     end
+
+    it "should consume all messages from plain 3-partition topic" do
+      kafka_input = LogStash::Inputs::Kafka.new(plain_config)
+      queue = Array.new
+      t = thread_it(kafka_input, queue)
+      t.run
+      wait(timeout_seconds).for { queue.length }.to eq(num_events)
+      expect(queue.length).to eq(num_events)
+    end
+
+    it "should consume all messages from snappy 3-partition topic" do
+      kafka_input = LogStash::Inputs::Kafka.new(snappy_config)
+      queue = Array.new
+      t = thread_it(kafka_input, queue)
+      t.run
+      wait(timeout_seconds).for { queue.length }.to eq(num_events)
+      expect(queue.length).to eq(num_events)
+    end
+
+    it "should consume all messages from lz4 3-partition topic" do
+      kafka_input = LogStash::Inputs::Kafka.new(lz4_config)
+      queue = Array.new
+      t = thread_it(kafka_input, queue)
+      t.run
+      wait(timeout_seconds).for { queue.length }.to eq(num_events)
+      expect(queue.length).to eq(num_events)
+    end
+    
   end
-  
-  def wait_for_events(queue, num_events)
-    begin
-      timeout(30) do
-        until queue.length == num_events do
-          sleep 1
-          next
+
+  describe "#kafka-topics-pattern" do
+    
+    def thread_it(kafka_input, queue)
+      Thread.new do
+        begin
+          kafka_input.run(queue)
         end
       end
     end
-  end  
-    
-  it "should consume all messages from 3-partition topic" do
-    kafka_input = LogStash::Inputs::Kafka.new(partition3_config)
-    queue = Array.new
-    t = thread_it(kafka_input, queue)
-    t.run
-    wait_for_events(queue, num_events)
-    expect(queue.size).to eq(num_events)
-  end
-  
-  it "should consume all messages from snappy 3-partition topic" do
-    kafka_input = LogStash::Inputs::Kafka.new(snappy_config)
-    queue = Array.new
-    t = thread_it(kafka_input, queue)
-    t.run
-    wait_for_events(queue, num_events)
-    expect(queue.size).to eq(num_events)
+      
+    it "should consume all messages from all 3 topics" do
+      kafka_input = LogStash::Inputs::Kafka.new(pattern_config)
+      queue = Array.new
+      t = thread_it(kafka_input, queue)
+      t.run
+      wait(timeout_seconds).for { queue.length }.to eq(3*num_events)
+      expect(queue.length).to eq(3*num_events)
+    end    
   end
 
-  it "should consume all messages from lz4 3-partition topic" do
-    kafka_input = LogStash::Inputs::Kafka.new(lz4_config)
-    queue = Array.new
-    t = thread_it(kafka_input, queue)
-    t.run
-    wait_for_events(queue, num_events)
-    expect(queue.size).to eq(num_events)
+  describe "#kafka-decorate" do
+    def thread_it(kafka_input, queue)
+      Thread.new do
+        begin
+          kafka_input.run(queue)
+        end
+      end
+    end
+      
+    it "should show the right topic and group name in decorated kafka section" do
+      kafka_input = LogStash::Inputs::Kafka.new(decorate_config)
+      queue = Queue.new
+      t = thread_it(kafka_input, queue)
+      t.run
+      wait(timeout_seconds).for { queue.length }.to eq(num_events)
+      expect(queue.length).to eq(num_events)
+      event = queue.shift
+      expect(event.get("kafka")["topic"]).to eq("logstash_topic_plain")
+      expect(event.get("kafka")["consumer_group"]).to eq(group_id_3)
+    end
   end
-  
 end

data/spec/unit/inputs/kafka_spec.rb

@@ -16,7 +16,7 @@ class MockConsumer
       raise org.apache.kafka.common.errors.WakeupException.new
     else
       10.times.map do
-        org.apache.kafka.clients.consumer.ConsumerRecord.new("test", 0, 0, "key", "value")
+        org.apache.kafka.clients.consumer.ConsumerRecord.new("logstash", 0, 0, "key", "value")
       end
     end
   end
@@ -30,7 +30,7 @@ class MockConsumer
 end
 
 describe LogStash::Inputs::Kafka do
-  let(:config) { { 'topics' => ['test'], 'consumer_threads' => 4 } }
+  let(:config) { { 'topics' => ['logstash'], 'consumer_threads' => 4 } }
   subject { LogStash::Inputs::Kafka.new(config) }
 
   it "should register" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-kafka
 version: !ruby/object:Gem::Version
-  version: 4.1.1
+  version: 4.2.0
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-11-14 00:00:00.000000000 Z
+date: 2017-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement