logstash-input-kafka 6.3.0 → 6.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 67d7e39487de32bc46086f71dc728d0b73a8d51d
-  data.tar.gz: 3e95c206a4142c3c90aa814e33d534edf5a6ce94
+  metadata.gz: b06b71f4df8ecdda42aed0a84edd1b3628e8d9b3
+  data.tar.gz: 7d90da1ec51686b3df424651784eaefb933b0676
 SHA512:
-  metadata.gz: aba387cb731bc155f245f6fd913b4cdd41b0cd9ef39468f392412fb4424b9474bf01f4156465115eb94860d1a22cf8c60a2dcae81be699b30a0d91ce5ba4bb70
-  data.tar.gz: aaebae3fb793842fca36c652c3de82812bbb6811a82d7d3ad6e29ebb3c9de38ebf14fd40e6eb5c14f37ac89b39d244de4991bcb6c605de055810da71094212d7
+  metadata.gz: 8dcf8af78c905b20ef38506c7a16cc4f09065065d2ec3351cc3a9343fb02539142351e53e98063aa5182a6561275eb30ae0b520c5601cec38fbe121bf89a4f67
+  data.tar.gz: 4363abfff3ad562071a4093bd925001ea591dba5f902cd3f724b539c2f4f779daa604d4288d10335e8556b242ac5d79e11ca56602f340d0538cfbc02f7c76955

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 6.3.1
+  - fix: Added record timestamp in event decoration
+
 ## 6.3.0
   - Upgrade Kafka client to version 0.10.2.1
 

data/Gemfile

@@ -1,3 +1,11 @@
 source 'https://rubygems.org'
 
 gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
+use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+
+if Dir.exist?(logstash_path) && use_logstash_source
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/README.md

@@ -6,6 +6,15 @@ This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
 
+## Logging
+
+Kafka logs do not respect the Log4J2 root logger level and defaults to INFO, for other levels, you must explicitly set the log level in your Logstash deployment's `log4j2.properties` file, e.g.:
+```
+logger.kafka.name=org.apache.kafka
+logger.kafka.appenderRef.console.ref=console
+logger.kafka.level=debug
+```
+
 ## Documentation
 
 https://www.elastic.co/guide/en/logstash/current/plugins-inputs-kafka.html

data/docs/index.asciidoc

@@ -7,14 +7,14 @@ START - GENERATED VARIABLES, DO NOT EDIT!
 :version: %VERSION%
 :release_date: %RELEASE_DATE%
 :changelog_url: %CHANGELOG_URL%
-:include_path: ../../../logstash/docs/include
+:include_path: ../../../../logstash/docs/include
 ///////////////////////////////////////////
 END - GENERATED VARIABLES, DO NOT EDIT!
 ///////////////////////////////////////////
 
 [id="plugins-{type}-{plugin}"]
 
-=== Kafka
+=== Kafka input plugin
 
 include::{include_path}/plugin_header.asciidoc[]
 
@@ -67,7 +67,7 @@ Kafka consumer configuration: http://kafka.apache.org/documentation.html#consume
 [id="plugins-{type}s-{plugin}-options"]
 ==== Kafka Input Configuration Options
 
-This plugin supports the following configuration options plus the <<plugins-{type}s-common-options>> described later.
+This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
 
 [cols="<,<,<",options="header",]
 |=======================================================================
@@ -117,7 +117,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-value_deserializer_class>> |<<string,string>>|No
 |=======================================================================
 
-Also see <<plugins-{type}s-common-options>> for a list of options supported by all
+Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
 input plugins.
 
 &nbsp;
@@ -548,4 +548,5 @@ Java Class used to deserialize the record's value
 
 
 
-include::{include_path}/{type}.asciidoc[]
+[id="plugins-{type}s-{plugin}-common-options"]
+include::{include_path}/{type}.asciidoc[]

data/lib/logstash/inputs/kafka.rb

@@ -210,6 +210,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   #   `partition`: The partition this message is associated with
   #   `offset`: The offset from the partition this message is associated with
   #   `key`: A ByteBuffer containing the message key
+  #   `timestamp`: The timestamp of this message
   config :decorate_events, :validate => :boolean, :default => false
 
 
@@ -257,6 +258,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
                 event.set("[kafka][partition]", record.partition)
                 event.set("[kafka][offset]", record.offset)
                 event.set("[kafka][key]", record.key)
+                event.set("[kafka][timestamp]", record.timestamp)
               end
               logstash_queue << event
             end

data/logstash-input-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-kafka'
-  s.version         = '6.3.0'
+  s.version         = '6.3.2'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = 'This input will read events from a Kafka topic. It uses the high level consumer API provided by Kafka to read messages from the broker'
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -19,8 +19,7 @@ Gem::Specification.new do |s|
   s.metadata = { 'logstash_plugin' => 'true', 'group' => 'input'}
 
   s.requirements << "jar 'org.apache.kafka:kafka-clients', '0.10.2.1'"
-  s.requirements << "jar 'org.slf4j:slf4j-log4j12', '1.7.21'"
-  s.requirements << "jar 'org.apache.logging.log4j:log4j-1.2-api', '2.6.2'"
+  s.requirements << "jar 'org.apache.logging.log4j:log4j-slf4j-impl', '2.8.2'"
 
   s.add_development_dependency 'jar-dependencies', '~> 0.3.2'
 

data/spec/integration/inputs/kafka_spec.rb

@@ -11,14 +11,15 @@ describe "inputs/kafka", :integration => true do
   let(:group_id_2) {rand(36**8).to_s(36)}
   let(:group_id_3) {rand(36**8).to_s(36)}
   let(:group_id_4) {rand(36**8).to_s(36)}
+  let(:group_id_5) {rand(36**8).to_s(36)}
   let(:plain_config) { { 'topics' => ['logstash_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'} }
   let(:multi_consumer_config) { plain_config.merge({"group_id" => group_id_4, "client_id" => "spec", "consumer_threads" => 3}) }
   let(:snappy_config) { { 'topics' => ['logstash_topic_snappy'], 'codec' => 'plain', 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'} }
   let(:lz4_config) { { 'topics' => ['logstash_topic_lz4'], 'codec' => 'plain', 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'} }
   let(:pattern_config) { { 'topics_pattern' => 'logstash_topic_.*', 'group_id' => group_id_2, 'codec' => 'plain', 'auto_offset_reset' => 'earliest'} }  
   let(:decorate_config) { { 'topics' => ['logstash_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_3, 'auto_offset_reset' => 'earliest', 'decorate_events' => true} }
-  let(:manual_commit_config) { { 'topics' => ['logstash_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_4, 'auto_offset_reset' => 'earliest', 'enable_auto_commit' => 'false'} }
-  let(:timeout_seconds) { 120 }
+  let(:manual_commit_config) { { 'topics' => ['logstash_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_5, 'auto_offset_reset' => 'earliest', 'enable_auto_commit' => 'false'} }
+  let(:timeout_seconds) { 30 }
   let(:num_events) { 103 }
 
   describe "#kafka-topics" do
@@ -32,40 +33,60 @@ describe "inputs/kafka", :integration => true do
 
     it "should consume all messages from plain 3-partition topic" do
       kafka_input = LogStash::Inputs::Kafka.new(plain_config)
-      queue = Array.new
+      queue = Queue.new
       t = thread_it(kafka_input, queue)
-      t.run
-      wait(timeout_seconds).for { queue.length }.to eq(num_events)
-      expect(queue.length).to eq(num_events)
+      begin
+        t.run
+        wait(timeout_seconds).for {queue.length}.to eq(num_events)
+        expect(queue.length).to eq(num_events)
+      ensure
+        t.kill
+        t.join(30_000)
+      end
     end
 
     it "should consume all messages from snappy 3-partition topic" do
       kafka_input = LogStash::Inputs::Kafka.new(snappy_config)
-      queue = Array.new
+      queue = Queue.new
       t = thread_it(kafka_input, queue)
-      t.run
-      wait(timeout_seconds).for { queue.length }.to eq(num_events)
-      expect(queue.length).to eq(num_events)
+      begin
+        t.run
+        wait(timeout_seconds).for {queue.length}.to eq(num_events)
+        expect(queue.length).to eq(num_events)
+      ensure
+        t.kill
+        t.join(30_000)
+      end
     end
 
     it "should consume all messages from lz4 3-partition topic" do
       kafka_input = LogStash::Inputs::Kafka.new(lz4_config)
-      queue = Array.new
+      queue = Queue.new
       t = thread_it(kafka_input, queue)
-      t.run
-      wait(timeout_seconds).for { queue.length }.to eq(num_events)
-      expect(queue.length).to eq(num_events)
+      begin
+        t.run
+        wait(timeout_seconds).for {queue.length}.to eq(num_events)
+        expect(queue.length).to eq(num_events)
+      ensure
+        t.kill
+        t.join(30_000)
+      end
     end
 
     it "should consumer all messages with multiple consumers" do
       kafka_input = LogStash::Inputs::Kafka.new(multi_consumer_config)
-      queue = Array.new
+      queue = Queue.new
       t = thread_it(kafka_input, queue)
-      t.run
-      wait(timeout_seconds).for { queue.length }.to eq(num_events)
-      expect(queue.length).to eq(num_events)
-      kafka_input.kafka_consumers.each_with_index do |consumer, i|
-        expect(consumer.metrics.keys.first.tags["client-id"]).to eq("spec-#{i}")
+      begin
+        t.run
+        wait(timeout_seconds).for {queue.length}.to eq(num_events)
+        expect(queue.length).to eq(num_events)
+        kafka_input.kafka_consumers.each_with_index do |consumer, i|
+          expect(consumer.metrics.keys.first.tags["client-id"]).to eq("spec-#{i}")
+        end
+      ensure
+        t.kill
+        t.join(30_000)
       end
     end
   end
@@ -81,11 +102,16 @@ describe "inputs/kafka", :integration => true do
 
     it "should consume all messages from all 3 topics" do
       kafka_input = LogStash::Inputs::Kafka.new(pattern_config)
-      queue = Array.new
+      queue = Queue.new
       t = thread_it(kafka_input, queue)
-      t.run
-      wait(timeout_seconds).for { queue.length }.to eq(3*num_events)
-      expect(queue.length).to eq(3*num_events)
+      begin
+        t.run
+        wait(timeout_seconds).for {queue.length}.to eq(3*num_events)
+        expect(queue.length).to eq(3*num_events)
+      ensure
+        t.kill
+        t.join(30_000)
+      end
     end
   end
 
@@ -99,15 +125,22 @@ describe "inputs/kafka", :integration => true do
     end
 
     it "should show the right topic and group name in decorated kafka section" do
+      start = LogStash::Timestamp.now.time.to_i
       kafka_input = LogStash::Inputs::Kafka.new(decorate_config)
       queue = Queue.new
       t = thread_it(kafka_input, queue)
-      t.run
-      wait(timeout_seconds).for { queue.length }.to eq(num_events)
-      expect(queue.length).to eq(num_events)
-      event = queue.shift
-      expect(event.get("kafka")["topic"]).to eq("logstash_topic_plain")
-      expect(event.get("kafka")["consumer_group"]).to eq(group_id_3)
+      begin
+        t.run
+        wait(timeout_seconds).for {queue.length}.to eq(num_events)
+        expect(queue.length).to eq(num_events)
+        event = queue.shift
+        expect(event.get("kafka")["topic"]).to eq("logstash_topic_plain")
+        expect(event.get("kafka")["consumer_group"]).to eq(group_id_3)
+        expect(event.get("kafka")["timestamp"]).to be >= start
+      ensure
+        t.kill
+        t.join(30_000)
+      end
     end
   end
 
@@ -122,11 +155,16 @@ describe "inputs/kafka", :integration => true do
 
     it "should manually commit offsets" do
       kafka_input = LogStash::Inputs::Kafka.new(manual_commit_config)
-      queue = Array.new
+      queue = Queue.new
       t = thread_it(kafka_input, queue)
-      t.run
-      wait(timeout_seconds).for { queue.length }.to eq(num_events)
-      expect(queue.length).to eq(num_events)
+      begin
+        t.run
+        wait(timeout_seconds).for {queue.length}.to eq(num_events)
+        expect(queue.length).to eq(num_events)
+      ensure
+        t.kill
+        t.join(30_000)
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-kafka
 version: !ruby/object:Gem::Version
-  version: 6.3.0
+  version: 6.3.2
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-05-11 00:00:00.000000000 Z
+date: 2017-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -134,28 +134,22 @@ files:
 - NOTICE.TXT
 - README.md
 - docs/index.asciidoc
-- lib/log4j/log4j/1.2.17/log4j-1.2.17.jar
 - lib/logstash-input-kafka_jars.rb
 - lib/logstash/inputs/kafka.rb
 - lib/net/jpountz/lz4/lz4/1.3.0/lz4-1.3.0.jar
 - lib/org/apache/kafka/kafka-clients/0.10.2.1/kafka-clients-0.10.2.1.jar
-- lib/org/apache/logging/log4j/log4j-1.2-api/2.6.2/log4j-1.2-api-2.6.2.jar
-- lib/org/apache/logging/log4j/log4j-api/2.6.2/log4j-api-2.6.2.jar
-- lib/org/apache/logging/log4j/log4j-core/2.6.2/log4j-core-2.6.2.jar
+- lib/org/apache/logging/log4j/log4j-api/2.8.2/log4j-api-2.8.2.jar
+- lib/org/apache/logging/log4j/log4j-slf4j-impl/2.8.2/log4j-slf4j-impl-2.8.2.jar
 - lib/org/slf4j/slf4j-api/1.7.21/slf4j-api-1.7.21.jar
-- lib/org/slf4j/slf4j-log4j12/1.7.21/slf4j-log4j12-1.7.21.jar
 - lib/org/xerial/snappy/snappy-java/1.1.2.6/snappy-java-1.1.2.6.jar
 - logstash-input-kafka.gemspec
 - spec/integration/inputs/kafka_spec.rb
 - spec/unit/inputs/kafka_spec.rb
 - vendor/jar-dependencies/runtime-jars/kafka-clients-0.10.2.1.jar
-- vendor/jar-dependencies/runtime-jars/log4j-1.2-api-2.6.2.jar
-- vendor/jar-dependencies/runtime-jars/log4j-1.2.17.jar
-- vendor/jar-dependencies/runtime-jars/log4j-api-2.6.2.jar
-- vendor/jar-dependencies/runtime-jars/log4j-core-2.6.2.jar
+- vendor/jar-dependencies/runtime-jars/log4j-api-2.8.2.jar
+- vendor/jar-dependencies/runtime-jars/log4j-slf4j-impl-2.8.2.jar
 - vendor/jar-dependencies/runtime-jars/lz4-1.3.0.jar
 - vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.21.jar
-- vendor/jar-dependencies/runtime-jars/slf4j-log4j12-1.7.21.jar
 - vendor/jar-dependencies/runtime-jars/snappy-java-1.1.2.6.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
@@ -179,8 +173,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements:
 - jar 'org.apache.kafka:kafka-clients', '0.10.2.1'
-- jar 'org.slf4j:slf4j-log4j12', '1.7.21'
-- jar 'org.apache.logging.log4j:log4j-1.2-api', '2.6.2'
+- jar 'org.apache.logging.log4j:log4j-slf4j-impl', '2.8.2'
 rubyforge_project:
 rubygems_version: 2.4.8
 signing_key: