logstash-input-kafka 3.0.3 → 3.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 15340382c64806f572f5bd3deb5ef912d4cc0f96
-  data.tar.gz: e573c0a1b2168d2e4fe0346e9a7269d48e8bcaa8
+  metadata.gz: a987ea01376b0e65e4fd7e8cadcccf2b173f5f84
+  data.tar.gz: 8ddfd1408df8c2a3750bba4c00af7774a69357b1
 SHA512:
-  metadata.gz: 2fd86eae77621259aebc77480d4e092fb3ca79e5fa573e4916bbb2492dca53cbd2966c5d82e65904677c3eea4cd98c7619a8c86cb42d1993eeea466d136f2c7f
-  data.tar.gz: 4c55185651161fc07d6737c15e3ffe127f2ffb4437bba9d73c49bfb9362b39548a94575e7f8d85e6c308c20ad653cda4c98654f9b4ed577dfe2e559449dbd2ef
+  metadata.gz: abe122982f283fdc82c640171be896646f3b321ddfb7ff96a3c562a56788a049c1868b9e1e4432d895ea2680458da8ab47ac4b5d32b7d3bfc7204c530a165d2f
+  data.tar.gz: 48e2f000909d011655386798ddad46200ede382763a3eec13a716bd29264ffc0f488bd18588922bcda739c09f9e1080b0fbf83c8bc6cab82e695d55dce2f5e01

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 3.1.0
+  - bump to client version 0.9.0.1
+  - add back :decorate_events option
+  - update :topics to default to ["logstash"]
+  - add :topics_pattern configuration option
+
 ## 3.0.3
   - Revert back to not configuring log4j here in this plugin. This setup method used 
   will hardcode log4j setting which means you cannot configure it. Its better to add 

data/lib/logstash/inputs/kafka.rb

@@ -114,8 +114,11 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   config :session_timeout_ms, :validate => :string, :default => "30000"
   # Java Class used to deserialize the record's value
   config :value_deserializer_class, :validate => :string, :default => "org.apache.kafka.common.serialization.StringDeserializer"
-  # A list of topics to subscribe to.
-  config :topics, :validate => :array, :required => true
+  # A list of topics to subscribe to, defaults to ["logstash"].
+  config :topics, :validate => :array, :default => ["logstash"]
+  # A topic regex pattern to subscribe to. 
+  # The topics configuration will be ignored when using this configuration.
+  config :topics_pattern, :validate => :string
   # Time kafka consumer will wait to receive new messages from topics
   config :poll_timeout_ms, :validate => :number, :default => 100
   # Enable SSL/TLS secured communication to Kafka broker. Note that secure communication 
@@ -129,6 +132,14 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   config :ssl_keystore_location, :validate => :path
   # If client authentication is required, this setting stores the keystore password
   config :ssl_keystore_password, :validate => :password
+  # Option to add Kafka metadata like topic, message size to the event.
+  # This will add a field named `kafka` to the logstash event containing the following attributes:
+  #   `topic`: The topic this message is associated with
+  #   `consumer_group`: The consumer group used to read in this event
+  #   `partition`: The partition this message is associated with
+  #   `offset`: The offset from the partition this message is associated with
+  #   `key`: A ByteBuffer containing the message key
+  config :decorate_events, :validate => :boolean, :default => false
 
   
   public
@@ -152,12 +163,25 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   def thread_runner(logstash_queue, consumer)
     Thread.new do
       begin
-        consumer.subscribe(topics);
+        unless @topics_pattern.nil?
+          nooplistener = org.apache.kafka.clients.consumer.internals.NoOpConsumerRebalanceListener.new
+          pattern = java.util.regex.Pattern.compile(@topics_pattern)
+          consumer.subscribe(pattern, nooplistener)
+        else
+          consumer.subscribe(topics);
+        end
         while !stop?
           records = consumer.poll(poll_timeout_ms);
           for record in records do
             @codec.decode(record.value.to_s) do |event|
               decorate(event)
+              if @decorate_events
+                event.set("[kafka][topic]", record.topic)
+                event.set("[kafka][consumer_group]", @group_id)
+                event.set("[kafka][partition]", record.partition)
+                event.set("[kafka][offset]", record.offset)
+                event.set("[kafka][key]", record.key)
+              end
               logstash_queue << event
             end
           end

data/logstash-input-kafka.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-kafka'
-  s.version         = '3.0.3'
+  s.version         = '3.1.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = 'This input will read events from a Kafka topic. It uses the high level consumer API provided by Kafka to read messages from the broker'
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/integration/inputs/kafka_spec.rb

@@ -3,58 +3,97 @@ require "logstash/devutils/rspec/spec_helper"
 require "logstash/inputs/kafka"
 require "digest"
 
-describe "input/kafka", :integration => true do
-  let(:partition3_config) { { 'topics' => ['topic3'], 'codec' => 'plain', 'auto_offset_reset' => 'earliest'} }
-  let(:snappy_config) { { 'topics' => ['snappy_topic'], 'codec' => 'plain', 'auto_offset_reset' => 'earliest'} }
-  let(:lz4_config) { { 'topics' => ['lz4_topic'], 'codec' => 'plain', 'auto_offset_reset' => 'earliest'} }
-  
-  let(:tries) { 60 }
+# Please run kafka_test_setup.sh prior to executing this integration test.
+describe "inputs/kafka", :integration => true do
+  # Group ids to make sure that the consumers get all the logs.
+  let(:group_id_1) {rand(36**8).to_s(36)}
+  let(:group_id_2) {rand(36**8).to_s(36)}
+  let(:group_id_3) {rand(36**8).to_s(36)}
+  let(:plain_config) { { 'topics' => ['logstash_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'} }
+  let(:snappy_config) { { 'topics' => ['logstash_topic_snappy'], 'codec' => 'plain', 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'} }
+  let(:lz4_config) { { 'topics' => ['logstash_topic_lz4'], 'codec' => 'plain', 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'} }
+  let(:pattern_config) { { 'topics_pattern' => 'logstash_topic_.*', 'group_id' => group_id_2, 'codec' => 'plain', 'auto_offset_reset' => 'earliest'} }
+  let(:decorate_config) { { 'topics' => ['logstash_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_3, 'auto_offset_reset' => 'earliest', 'decorate_events' => true} }
+  let(:timeout_seconds) { 120 }
   let(:num_events) { 103 }
-  
-  def thread_it(kafka_input, queue)
-    Thread.new do
-      begin
-        kafka_input.run(queue)
+
+  describe "#kafka-topics" do
+    def thread_it(kafka_input, queue)
+      Thread.new do
+        begin
+          kafka_input.run(queue)
+        end
       end
     end
+
+    it "should consume all messages from plain 3-partition topic" do
+      kafka_input = LogStash::Inputs::Kafka.new(plain_config)
+      queue = Array.new
+      t = thread_it(kafka_input, queue)
+      t.run
+      wait(timeout_seconds).for { queue.length }.to eq(num_events)
+      expect(queue.length).to eq(num_events)
+    end
+
+    it "should consume all messages from snappy 3-partition topic" do
+      kafka_input = LogStash::Inputs::Kafka.new(snappy_config)
+      queue = Array.new
+      t = thread_it(kafka_input, queue)
+      t.run
+      wait(timeout_seconds).for { queue.length }.to eq(num_events)
+      expect(queue.length).to eq(num_events)
+    end
+
+    it "should consume all messages from lz4 3-partition topic" do
+      kafka_input = LogStash::Inputs::Kafka.new(lz4_config)
+      queue = Array.new
+      t = thread_it(kafka_input, queue)
+      t.run
+      wait(timeout_seconds).for { queue.length }.to eq(num_events)
+      expect(queue.length).to eq(num_events)
+    end
+
   end
-  
-  def wait_for_events(queue, num_events)
-    begin
-      timeout(30) do
-        until queue.length == num_events do
-          sleep 1
-          next
+
+  describe "#kafka-topics-pattern" do
+
+    def thread_it(kafka_input, queue)
+      Thread.new do
+        begin
+          kafka_input.run(queue)
         end
       end
     end
-  end  
-    
-  it "should consume all messages from 3-partition topic" do
-    kafka_input = LogStash::Inputs::Kafka.new(partition3_config)
-    queue = Array.new
-    t = thread_it(kafka_input, queue)
-    t.run
-    wait_for_events(queue, num_events)
-    expect(queue.size).to eq(num_events)
-  end
-  
-  it "should consume all messages from snappy 3-partition topic" do
-    kafka_input = LogStash::Inputs::Kafka.new(snappy_config)
-    queue = Array.new
-    t = thread_it(kafka_input, queue)
-    t.run
-    wait_for_events(queue, num_events)
-    expect(queue.size).to eq(num_events)
+
+    it "should consume all messages from all 3 topics" do
+      kafka_input = LogStash::Inputs::Kafka.new(pattern_config)
+      queue = Array.new
+      t = thread_it(kafka_input, queue)
+      t.run
+      wait(timeout_seconds).for { queue.length }.to eq(3*num_events)
+      expect(queue.length).to eq(3*num_events)
+    end
   end
 
-  it "should consume all messages from lz4 3-partition topic" do
-    kafka_input = LogStash::Inputs::Kafka.new(lz4_config)
-    queue = Array.new
-    t = thread_it(kafka_input, queue)
-    t.run
-    wait_for_events(queue, num_events)
-    expect(queue.size).to eq(num_events)
+  describe "#kafka-decorate" do
+    def thread_it(kafka_input, queue)
+      Thread.new do
+        begin
+          kafka_input.run(queue)
+        end
+      end
+    end
+
+    it "should show the right topic and group name in decorated kafka section" do
+      kafka_input = LogStash::Inputs::Kafka.new(decorate_config)
+      queue = Queue.new
+      t = thread_it(kafka_input, queue)
+      t.run
+      wait(timeout_seconds).for { queue.length }.to eq(num_events)
+      expect(queue.length).to eq(num_events)
+      event = queue.shift
+      expect(event.get("kafka")["topic"]).to eq("logstash_topic_plain")
+      expect(event.get("kafka")["consumer_group"]).to eq(group_id_3)
+    end
   end
-  
 end

data/spec/unit/inputs/kafka_spec.rb

@@ -16,7 +16,7 @@ class MockConsumer
       raise org.apache.kafka.common.errors.WakeupException.new
     else
       10.times.map do
-        org.apache.kafka.clients.consumer.ConsumerRecord.new("test", 0, 0, "key", "value")
+        org.apache.kafka.clients.consumer.ConsumerRecord.new("logstash", 0, 0, "key", "value")
       end
     end
   end
@@ -30,7 +30,7 @@ class MockConsumer
 end
 
 describe LogStash::Inputs::Kafka do
-  let(:config) { { 'topics' => ['test'], 'consumer_threads' => 4 } }
+  let(:config) { { 'topics' => ['logstash'], 'consumer_threads' => 4 } }
   subject { LogStash::Inputs::Kafka.new(config) }
 
   it "should register" do

metadata

@@ -1,19 +1,19 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-kafka
 version: !ruby/object:Gem::Version
-  version: 3.0.3
+  version: 3.1.0
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-06-03 00:00:00.000000000 Z
+date: 2016-11-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.3.2
   name: jar-dependencies
@@ -21,13 +21,13 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.3.2
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
   name: logstash-core-plugin-api
@@ -35,13 +35,13 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-codec-json
@@ -49,13 +49,13 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-codec-plain
@@ -63,16 +63,16 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.0.22
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: 0.1.0
   name: stud
@@ -80,16 +80,16 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.0.22
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: 0.1.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-devutils
@@ -97,7 +97,7 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
@@ -118,12 +118,6 @@ files:
 - logstash-input-kafka.gemspec
 - spec/integration/inputs/kafka_spec.rb
 - spec/unit/inputs/kafka_spec.rb
-- vendor/jar-dependencies/runtime-jars/kafka-clients-0.9.0.1.jar
-- vendor/jar-dependencies/runtime-jars/log4j-1.2.17.jar
-- vendor/jar-dependencies/runtime-jars/lz4-1.2.0.jar
-- vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.6.jar
-- vendor/jar-dependencies/runtime-jars/slf4j-log4j12-1.7.13.jar
-- vendor/jar-dependencies/runtime-jars/snappy-java-1.1.1.7.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -136,19 +130,19 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements:
 - jar 'org.apache.kafka:kafka-clients', '0.9.0.1'
 - jar 'org.slf4j:slf4j-log4j12', '1.7.13'
 rubyforge_project:
-rubygems_version: 2.4.5
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: This input will read events from a Kafka topic. It uses the high level consumer API provided by Kafka to read messages from the broker