logstash-input-kafka 5.0.1 → 5.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f496e94c2acee0bbb82357e17efe952416381c4c
-  data.tar.gz: 19279054c88de9ecc57dedab85c1e584afd9f160
+  metadata.gz: 1e36a1e371b9818c8112475405e598724b548818
+  data.tar.gz: fb077840636c3ef8f466eafc3c8d6304f2a1bc75
 SHA512:
-  metadata.gz: 6b3ae71f305fe402aa9dbafdb815ace949445e90168b310f2b35f90cf68d83d5b4ec69389e61d3950e897b3b8c62f24126cc2995f98b6ada301b50641d495a00
-  data.tar.gz: 3eb69a86fe5377bc4d348b601db819b76d1ae296ec44fe761a98db2629eae194a34590612f939f88e270d6cf5fc66b227d5ba97cc2c2ad5e129eee6944f863ff
+  metadata.gz: b03b3177c9c875c39f0e21a3408058a3f54d160e91190c19bebd8526b837fc0abaf41a575249e1d1f681f631535a6e88fe8896bca62f9dfd80cdce6a46ce1154
+  data.tar.gz: 4a4fe642ca9fbfc44885646c4b5374d9d74091c6529a8df175af69b95f1510b92e8cc66065419ea3943bc7f8308e6c125f506a1795ab325a059b423d0e120ff9

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 5.0.2
+  - Release a new version of the gem that includes jars
+
 ## 5.0.1
   - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
 

data/README.md

@@ -17,9 +17,10 @@ Here's a table that describes the compatibility matrix for Kafka Broker support.
 | 0.9          | 2.4, 5.0           |   4.0.0 | 4.0.0  | Intermediate release before 0.10 with new get/set API |
 | 0.10         | 2.4, 5.0           |   5.0.0 | 5.0.0  | Track latest Kafka release. Not compatible with 0.9 broker |
 
-
 ## Documentation
 
+https://www.elastic.co/guide/en/logstash/current/plugins-inputs-kafka.html
+
 Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.co/guide/en/logstash/current/).
 
 - For formatting code or config example, you can use the asciidoc `[source,ruby]` directive
@@ -41,6 +42,7 @@ Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/log
 - Install dependencies
 ```sh
 bundle install
+rake install_jars
 ```
 
 #### Test
@@ -49,6 +51,7 @@ bundle install
 
 ```sh
 bundle install
+rake install_jars
 ```
 
 - Run tests

data/lib/logstash/inputs/kafka.rb

@@ -136,8 +136,11 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   config :session_timeout_ms, :validate => :string
   # Java Class used to deserialize the record's value
   config :value_deserializer_class, :validate => :string, :default => "org.apache.kafka.common.serialization.StringDeserializer"
-  # A list of topics to subscribe to.
-  config :topics, :validate => :array, :required => true
+  # A list of topics to subscribe to, defaults to ["logstash"].
+  config :topics, :validate => :array, :default => ["logstash"]
+  # A topic regex pattern to subscribe to. 
+  # The topics configuration will be ignored when using this configuration.
+  config :topics_pattern, :validate => :string
   # Time kafka consumer will wait to receive new messages from topics
   config :poll_timeout_ms, :validate => :number
   # Enable SSL/TLS secured communication to Kafka broker.
@@ -150,6 +153,14 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   config :ssl_keystore_location, :validate => :path
   # If client authentication is required, this setting stores the keystore password
   config :ssl_keystore_password, :validate => :password
+  # Option to add Kafka metadata like topic, message size to the event.
+  # This will add a field named `kafka` to the logstash event containing the following attributes:
+  #   `topic`: The topic this message is associated with
+  #   `consumer_group`: The consumer group used to read in this event
+  #   `partition`: The partition this message is associated with
+  #   `offset`: The offset from the partition this message is associated with
+  #   `key`: A ByteBuffer containing the message key
+  config :decorate_events, :validate => :boolean, :default => false
 
 
   public
@@ -174,12 +185,25 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   def thread_runner(logstash_queue, consumer)
     Thread.new do
       begin
-        consumer.subscribe(topics);
+        unless @topics_pattern.nil?
+          nooplistener = org.apache.kafka.clients.consumer.internals.NoOpConsumerRebalanceListener.new
+          pattern = java.util.regex.Pattern.compile(@topics_pattern)
+          consumer.subscribe(pattern, nooplistener)
+        else
+          consumer.subscribe(topics);
+        end
         while !stop?
           records = consumer.poll(poll_timeout_ms);
           for record in records do
             @codec.decode(record.value.to_s) do |event|
               decorate(event)
+              if @decorate_events
+                event.set("[kafka][topic]", record.topic)
+                event.set("[kafka][consumer_group]", @group_id)
+                event.set("[kafka][partition]", record.partition)
+                event.set("[kafka][offset]", record.offset)
+                event.set("[kafka][key]", record.key)
+              end
               logstash_queue << event
             end
           end

data/logstash-input-kafka.gemspec

@@ -1,7 +1,6 @@
 Gem::Specification.new do |s|
-
   s.name            = 'logstash-input-kafka'
-  s.version         = '5.0.1'
+  s.version         = '5.0.2'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = 'This input will read events from a Kafka topic. It uses the high level consumer API provided by Kafka to read messages from the broker'
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/integration/inputs/kafka_spec.rb

@@ -4,47 +4,97 @@ require "logstash/inputs/kafka"
 require "digest"
 require "rspec/wait"
 
-describe "input/kafka", :integration => true do
-  let(:partition3_config) { { 'topics' => ['topic3'], 'codec' => 'plain', 'auto_offset_reset' => 'earliest'} }
-  let(:snappy_config) { { 'topics' => ['snappy_topic'], 'codec' => 'plain', 'auto_offset_reset' => 'earliest'} }
-  let(:lz4_config) { { 'topics' => ['lz4_topic'], 'codec' => 'plain', 'auto_offset_reset' => 'earliest'} }
-  
-  let(:timeout_seconds) { 3600 }
+# Please run kafka_test_setup.sh prior to executing this integration test.
+describe "inputs/kafka", :integration => true do
+  # Group ids to make sure that the consumers get all the logs.
+  let(:group_id_1) {rand(36**8).to_s(36)}
+  let(:group_id_2) {rand(36**8).to_s(36)}
+  let(:group_id_3) {rand(36**8).to_s(36)}
+  let(:plain_config) { { 'topics' => ['logstash_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'} }
+  let(:snappy_config) { { 'topics' => ['logstash_topic_snappy'], 'codec' => 'plain', 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'} }
+  let(:lz4_config) { { 'topics' => ['logstash_topic_lz4'], 'codec' => 'plain', 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'} }
+  let(:pattern_config) { { 'topics_pattern' => 'logstash_topic_.*', 'group_id' => group_id_2, 'codec' => 'plain', 'auto_offset_reset' => 'earliest'} }  
+  let(:decorate_config) { { 'topics' => ['logstash_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_3, 'auto_offset_reset' => 'earliest', 'decorate_events' => true} }
+  let(:timeout_seconds) { 120 }
   let(:num_events) { 103 }
-  
-  def thread_it(kafka_input, queue)
-    Thread.new do
-      begin
-        kafka_input.run(queue)
+
+  describe "#kafka-topics" do
+    def thread_it(kafka_input, queue)
+      Thread.new do
+        begin
+          kafka_input.run(queue)
+        end
       end
     end
-  end
+
+    it "should consume all messages from plain 3-partition topic" do
+      kafka_input = LogStash::Inputs::Kafka.new(plain_config)
+      queue = Array.new
+      t = thread_it(kafka_input, queue)
+      t.run
+      wait(timeout_seconds).for { queue.length }.to eq(num_events)
+      expect(queue.length).to eq(num_events)
+    end
+
+    it "should consume all messages from snappy 3-partition topic" do
+      kafka_input = LogStash::Inputs::Kafka.new(snappy_config)
+      queue = Array.new
+      t = thread_it(kafka_input, queue)
+      t.run
+      wait(timeout_seconds).for { queue.length }.to eq(num_events)
+      expect(queue.length).to eq(num_events)
+    end
+
+    it "should consume all messages from lz4 3-partition topic" do
+      kafka_input = LogStash::Inputs::Kafka.new(lz4_config)
+      queue = Array.new
+      t = thread_it(kafka_input, queue)
+      t.run
+      wait(timeout_seconds).for { queue.length }.to eq(num_events)
+      expect(queue.length).to eq(num_events)
+    end
     
-  it "should consume all messages from 3-partition topic" do
-    kafka_input = LogStash::Inputs::Kafka.new(partition3_config)
-    queue = Array.new
-    t = thread_it(kafka_input, queue)
-    t.run
-    wait(timeout_seconds).for { queue.length }.to eq(num_events)
-    expect(queue.length).to eq(num_events)
   end
-  
-  it "should consume all messages from snappy 3-partition topic" do
-    kafka_input = LogStash::Inputs::Kafka.new(snappy_config)
-    queue = Array.new
-    t = thread_it(kafka_input, queue)
-    t.run
-    wait(timeout_seconds).for { queue.length }.to eq(num_events)
-    expect(queue.length).to eq(num_events)
+
+  describe "#kafka-topics-pattern" do
+    
+    def thread_it(kafka_input, queue)
+      Thread.new do
+        begin
+          kafka_input.run(queue)
+        end
+      end
+    end
+      
+    it "should consume all messages from all 3 topics" do
+      kafka_input = LogStash::Inputs::Kafka.new(pattern_config)
+      queue = Array.new
+      t = thread_it(kafka_input, queue)
+      t.run
+      wait(timeout_seconds).for { queue.length }.to eq(3*num_events)
+      expect(queue.length).to eq(3*num_events)
+    end    
   end
 
-  it "should consume all messages from lz4 3-partition topic" do
-    kafka_input = LogStash::Inputs::Kafka.new(lz4_config)
-    queue = Array.new
-    t = thread_it(kafka_input, queue)
-    t.run
-    wait(timeout_seconds).for { queue.length }.to eq(num_events)
-    expect(queue.length).to eq(num_events)
+  describe "#kafka-decorate" do
+    def thread_it(kafka_input, queue)
+      Thread.new do
+        begin
+          kafka_input.run(queue)
+        end
+      end
+    end
+      
+    it "should show the right topic and group name in decorated kafka section" do
+      kafka_input = LogStash::Inputs::Kafka.new(decorate_config)
+      queue = Queue.new
+      t = thread_it(kafka_input, queue)
+      t.run
+      wait(timeout_seconds).for { queue.length }.to eq(num_events)
+      expect(queue.length).to eq(num_events)
+      event = queue.shift
+      expect(event.get("kafka")["topic"]).to eq("logstash_topic_plain")
+      expect(event.get("kafka")["consumer_group"]).to eq(group_id_3)
+    end
   end
-  
 end

data/spec/unit/inputs/kafka_spec.rb

@@ -16,7 +16,7 @@ class MockConsumer
       raise org.apache.kafka.common.errors.WakeupException.new
     else
       10.times.map do
-        org.apache.kafka.clients.consumer.ConsumerRecord.new("test", 0, 0, "key", "value")
+        org.apache.kafka.clients.consumer.ConsumerRecord.new("logstash", 0, 0, "key", "value")
       end
     end
   end
@@ -30,7 +30,7 @@ class MockConsumer
 end
 
 describe LogStash::Inputs::Kafka do
-  let(:config) { { 'topics' => ['test'], 'consumer_threads' => 4 } }
+  let(:config) { { 'topics' => ['logstash'], 'consumer_threads' => 4 } }
   subject { LogStash::Inputs::Kafka.new(config) }
 
   it "should register" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-kafka
 version: !ruby/object:Gem::Version
-  version: 5.0.1
+  version: 5.0.2
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-07-14 00:00:00.000000000 Z
+date: 2016-08-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -133,11 +133,23 @@ files:
 - LICENSE
 - NOTICE.TXT
 - README.md
+- lib/log4j/log4j/1.2.17/log4j-1.2.17.jar
 - lib/logstash-input-kafka_jars.rb
 - lib/logstash/inputs/kafka.rb
+- lib/net/jpountz/lz4/lz4/1.3.0/lz4-1.3.0.jar
+- lib/org/apache/kafka/kafka-clients/0.10.0.0/kafka-clients-0.10.0.0.jar
+- lib/org/slf4j/slf4j-api/1.7.21/slf4j-api-1.7.21.jar
+- lib/org/slf4j/slf4j-log4j12/1.7.21/slf4j-log4j12-1.7.21.jar
+- lib/org/xerial/snappy/snappy-java/1.1.2.4/snappy-java-1.1.2.4.jar
 - logstash-input-kafka.gemspec
 - spec/integration/inputs/kafka_spec.rb
 - spec/unit/inputs/kafka_spec.rb
+- vendor/jar-dependencies/runtime-jars/kafka-clients-0.10.0.0.jar
+- vendor/jar-dependencies/runtime-jars/log4j-1.2.17.jar
+- vendor/jar-dependencies/runtime-jars/lz4-1.3.0.jar
+- vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.21.jar
+- vendor/jar-dependencies/runtime-jars/slf4j-log4j12-1.7.21.jar
+- vendor/jar-dependencies/runtime-jars/snappy-java-1.1.2.4.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -162,7 +174,7 @@ requirements:
 - jar 'org.apache.kafka:kafka-clients', '0.10.0.0'
 - jar 'org.slf4j:slf4j-log4j12', '1.7.21'
 rubyforge_project:
-rubygems_version: 2.6.3
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: This input will read events from a Kafka topic. It uses the high level consumer API provided by Kafka to read messages from the broker