logstash-input-kafka 3.1.0 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a987ea01376b0e65e4fd7e8cadcccf2b173f5f84
-  data.tar.gz: 8ddfd1408df8c2a3750bba4c00af7774a69357b1
+  metadata.gz: abc3fc0897258da0fa0b331447bfb1f8a0d464cb
+  data.tar.gz: 0dd2bfc124413da3c415b8e87db660a155a5c8b4
 SHA512:
-  metadata.gz: abe122982f283fdc82c640171be896646f3b321ddfb7ff96a3c562a56788a049c1868b9e1e4432d895ea2680458da8ab47ac4b5d32b7d3bfc7204c530a165d2f
-  data.tar.gz: 48e2f000909d011655386798ddad46200ede382763a3eec13a716bd29264ffc0f488bd18588922bcda739c09f9e1080b0fbf83c8bc6cab82e695d55dce2f5e01
+  metadata.gz: 615fa1e2452d73baf95aadf248aa46961a1d9d2ba7c217c0ea1fd3323d74250b8135fb478a9262601166212f622ebf78a94a9bb88547f7bf0c6ffb465f9433ed
+  data.tar.gz: 78a16c4a6f7ad5037e8357e24a0638929d09484fa2df32e71ae5d568a93be5c75018557f64756969751782fff00674fd31809727825465dee61749f980e44ddf

data/CHANGELOG.md

@@ -1,17 +1,7 @@
-## 3.1.0
-  - bump to client version 0.9.0.1
-  - add back :decorate_events option
-  - update :topics to default to ["logstash"]
-  - add :topics_pattern configuration option
-
-## 3.0.3
-  - Revert back to not configuring log4j here in this plugin. This setup method used 
-  will hardcode log4j setting which means you cannot configure it. Its better to add 
-  log4j.properties in LS_HOME/bin, add it to classpath to silence warnings.
-  - Update auto_commit_interval_ms to default to 5000
-
-## 3.0.2
-  - Support for Kafka 0.9 for LS 2.x
+## 4.0.0
+  - Republish all the gems under jruby.
+  - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
+  - Support for Kafka 0.9 for LS 5.x
 
 ## 3.0.0.beta7
  - Fix Log4j warnings by setting up the logger
@@ -57,4 +47,3 @@
  - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
  - Dependency on logstash-core update to 2.0
- 

data/lib/logstash/inputs/kafka.rb

@@ -35,7 +35,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   default :codec, 'plain'
 
   # The frequency in milliseconds that the consumer offsets are committed to Kafka.
-  config :auto_commit_interval_ms, :validate => :string, :default => "5000"
+  config :auto_commit_interval_ms, :validate => :string, :default => "10"
   # What to do when there is no initial offset in Kafka or if an offset is out of range:
   #
   # * earliest: automatically reset the offset to the earliest offset
@@ -114,11 +114,8 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   config :session_timeout_ms, :validate => :string, :default => "30000"
   # Java Class used to deserialize the record's value
   config :value_deserializer_class, :validate => :string, :default => "org.apache.kafka.common.serialization.StringDeserializer"
-  # A list of topics to subscribe to, defaults to ["logstash"].
-  config :topics, :validate => :array, :default => ["logstash"]
-  # A topic regex pattern to subscribe to. 
-  # The topics configuration will be ignored when using this configuration.
-  config :topics_pattern, :validate => :string
+  # A list of topics to subscribe to.
+  config :topics, :validate => :array, :required => true
   # Time kafka consumer will wait to receive new messages from topics
   config :poll_timeout_ms, :validate => :number, :default => 100
   # Enable SSL/TLS secured communication to Kafka broker. Note that secure communication 
@@ -132,18 +129,11 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   config :ssl_keystore_location, :validate => :path
   # If client authentication is required, this setting stores the keystore password
   config :ssl_keystore_password, :validate => :password
-  # Option to add Kafka metadata like topic, message size to the event.
-  # This will add a field named `kafka` to the logstash event containing the following attributes:
-  #   `topic`: The topic this message is associated with
-  #   `consumer_group`: The consumer group used to read in this event
-  #   `partition`: The partition this message is associated with
-  #   `offset`: The offset from the partition this message is associated with
-  #   `key`: A ByteBuffer containing the message key
-  config :decorate_events, :validate => :boolean, :default => false
 
   
   public
   def register
+    LogStash::Logger.setup_log4j(@logger)
     @runner_threads = []
   end # def register
 
@@ -163,25 +153,12 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   def thread_runner(logstash_queue, consumer)
     Thread.new do
       begin
-        unless @topics_pattern.nil?
-          nooplistener = org.apache.kafka.clients.consumer.internals.NoOpConsumerRebalanceListener.new
-          pattern = java.util.regex.Pattern.compile(@topics_pattern)
-          consumer.subscribe(pattern, nooplistener)
-        else
-          consumer.subscribe(topics);
-        end
+        consumer.subscribe(topics);
         while !stop?
           records = consumer.poll(poll_timeout_ms);
           for record in records do
             @codec.decode(record.value.to_s) do |event|
               decorate(event)
-              if @decorate_events
-                event.set("[kafka][topic]", record.topic)
-                event.set("[kafka][consumer_group]", @group_id)
-                event.set("[kafka][partition]", record.partition)
-                event.set("[kafka][offset]", record.offset)
-                event.set("[kafka][key]", record.key)
-              end
               logstash_queue << event
             end
           end

data/logstash-input-kafka.gemspec

@@ -1,10 +1,10 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-kafka'
-  s.version         = '3.1.0'
+  s.version         = '4.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = 'This input will read events from a Kafka topic. It uses the high level consumer API provided by Kafka to read messages from the broker'
-  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors         = ['Elasticsearch']
   s.email           = 'info@elastic.co'
   s.homepage        = "http://www.elastic.co/guide/en/logstash/current/index.html"
@@ -25,9 +25,11 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'jar-dependencies', '~> 0.3.2'
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
   s.add_runtime_dependency 'logstash-codec-json'
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'stud', '>= 0.0.22', '< 0.1.0'
+
   s.add_development_dependency 'logstash-devutils'
 end
+

data/spec/integration/inputs/kafka_spec.rb

@@ -3,97 +3,58 @@ require "logstash/devutils/rspec/spec_helper"
 require "logstash/inputs/kafka"
 require "digest"
 
-# Please run kafka_test_setup.sh prior to executing this integration test.
-describe "inputs/kafka", :integration => true do
-  # Group ids to make sure that the consumers get all the logs.
-  let(:group_id_1) {rand(36**8).to_s(36)}
-  let(:group_id_2) {rand(36**8).to_s(36)}
-  let(:group_id_3) {rand(36**8).to_s(36)}
-  let(:plain_config) { { 'topics' => ['logstash_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'} }
-  let(:snappy_config) { { 'topics' => ['logstash_topic_snappy'], 'codec' => 'plain', 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'} }
-  let(:lz4_config) { { 'topics' => ['logstash_topic_lz4'], 'codec' => 'plain', 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'} }
-  let(:pattern_config) { { 'topics_pattern' => 'logstash_topic_.*', 'group_id' => group_id_2, 'codec' => 'plain', 'auto_offset_reset' => 'earliest'} }
-  let(:decorate_config) { { 'topics' => ['logstash_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_3, 'auto_offset_reset' => 'earliest', 'decorate_events' => true} }
-  let(:timeout_seconds) { 120 }
+describe "input/kafka", :integration => true do
+  let(:partition3_config) { { 'topics' => ['topic3'], 'codec' => 'plain', 'auto_offset_reset' => 'earliest'} }
+  let(:snappy_config) { { 'topics' => ['snappy_topic'], 'codec' => 'plain', 'auto_offset_reset' => 'earliest'} }
+  let(:lz4_config) { { 'topics' => ['lz4_topic'], 'codec' => 'plain', 'auto_offset_reset' => 'earliest'} }
+  
+  let(:tries) { 60 }
   let(:num_events) { 103 }
-
-  describe "#kafka-topics" do
-    def thread_it(kafka_input, queue)
-      Thread.new do
-        begin
-          kafka_input.run(queue)
-        end
+  
+  def thread_it(kafka_input, queue)
+    Thread.new do
+      begin
+        kafka_input.run(queue)
       end
     end
-
-    it "should consume all messages from plain 3-partition topic" do
-      kafka_input = LogStash::Inputs::Kafka.new(plain_config)
-      queue = Array.new
-      t = thread_it(kafka_input, queue)
-      t.run
-      wait(timeout_seconds).for { queue.length }.to eq(num_events)
-      expect(queue.length).to eq(num_events)
-    end
-
-    it "should consume all messages from snappy 3-partition topic" do
-      kafka_input = LogStash::Inputs::Kafka.new(snappy_config)
-      queue = Array.new
-      t = thread_it(kafka_input, queue)
-      t.run
-      wait(timeout_seconds).for { queue.length }.to eq(num_events)
-      expect(queue.length).to eq(num_events)
-    end
-
-    it "should consume all messages from lz4 3-partition topic" do
-      kafka_input = LogStash::Inputs::Kafka.new(lz4_config)
-      queue = Array.new
-      t = thread_it(kafka_input, queue)
-      t.run
-      wait(timeout_seconds).for { queue.length }.to eq(num_events)
-      expect(queue.length).to eq(num_events)
-    end
-
   end
-
-  describe "#kafka-topics-pattern" do
-
-    def thread_it(kafka_input, queue)
-      Thread.new do
-        begin
-          kafka_input.run(queue)
+  
+  def wait_for_events(queue, num_events)
+    begin
+      timeout(30) do
+        until queue.length == num_events do
+          sleep 1
+          next
         end
       end
     end
-
-    it "should consume all messages from all 3 topics" do
-      kafka_input = LogStash::Inputs::Kafka.new(pattern_config)
-      queue = Array.new
-      t = thread_it(kafka_input, queue)
-      t.run
-      wait(timeout_seconds).for { queue.length }.to eq(3*num_events)
-      expect(queue.length).to eq(3*num_events)
-    end
+  end  
+    
+  it "should consume all messages from 3-partition topic" do
+    kafka_input = LogStash::Inputs::Kafka.new(partition3_config)
+    queue = Array.new
+    t = thread_it(kafka_input, queue)
+    t.run
+    wait_for_events(queue, num_events)
+    expect(queue.size).to eq(num_events)
+  end
+  
+  it "should consume all messages from snappy 3-partition topic" do
+    kafka_input = LogStash::Inputs::Kafka.new(snappy_config)
+    queue = Array.new
+    t = thread_it(kafka_input, queue)
+    t.run
+    wait_for_events(queue, num_events)
+    expect(queue.size).to eq(num_events)
   end
 
-  describe "#kafka-decorate" do
-    def thread_it(kafka_input, queue)
-      Thread.new do
-        begin
-          kafka_input.run(queue)
-        end
-      end
-    end
-
-    it "should show the right topic and group name in decorated kafka section" do
-      kafka_input = LogStash::Inputs::Kafka.new(decorate_config)
-      queue = Queue.new
-      t = thread_it(kafka_input, queue)
-      t.run
-      wait(timeout_seconds).for { queue.length }.to eq(num_events)
-      expect(queue.length).to eq(num_events)
-      event = queue.shift
-      expect(event.get("kafka")["topic"]).to eq("logstash_topic_plain")
-      expect(event.get("kafka")["consumer_group"]).to eq(group_id_3)
-    end
+  it "should consume all messages from lz4 3-partition topic" do
+    kafka_input = LogStash::Inputs::Kafka.new(lz4_config)
+    queue = Array.new
+    t = thread_it(kafka_input, queue)
+    t.run
+    wait_for_events(queue, num_events)
+    expect(queue.size).to eq(num_events)
   end
+  
 end

data/spec/unit/inputs/kafka_spec.rb

@@ -16,7 +16,7 @@ class MockConsumer
       raise org.apache.kafka.common.errors.WakeupException.new
     else
       10.times.map do
-        org.apache.kafka.clients.consumer.ConsumerRecord.new("logstash", 0, 0, "key", "value")
+        org.apache.kafka.clients.consumer.ConsumerRecord.new("test", 0, 0, "key", "value")
       end
     end
   end
@@ -30,7 +30,7 @@ class MockConsumer
 end
 
 describe LogStash::Inputs::Kafka do
-  let(:config) { { 'topics' => ['logstash'], 'consumer_threads' => 4 } }
+  let(:config) { { 'topics' => ['test'], 'consumer_threads' => 4 } }
   subject { LogStash::Inputs::Kafka.new(config) }
 
   it "should register" do

metadata

@@ -1,19 +1,19 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-kafka
 version: !ruby/object:Gem::Version
-  version: 3.1.0
+  version: 4.0.0
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-11-16 00:00:00.000000000 Z
+date: 2016-05-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.3.2
   name: jar-dependencies
@@ -21,27 +21,27 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.3.2
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '2.0'
   name: logstash-core-plugin-api
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-codec-json
@@ -49,13 +49,13 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-codec-plain
@@ -63,16 +63,16 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.0.22
-    - - "<"
+    - - <
       - !ruby/object:Gem::Version
         version: 0.1.0
   name: stud
@@ -80,16 +80,16 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.0.22
-    - - "<"
+    - - <
       - !ruby/object:Gem::Version
         version: 0.1.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-devutils
@@ -97,10 +97,10 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
@@ -118,6 +118,12 @@ files:
 - logstash-input-kafka.gemspec
 - spec/integration/inputs/kafka_spec.rb
 - spec/unit/inputs/kafka_spec.rb
+- vendor/jar-dependencies/runtime-jars/kafka-clients-0.9.0.1.jar
+- vendor/jar-dependencies/runtime-jars/log4j-1.2.17.jar
+- vendor/jar-dependencies/runtime-jars/lz4-1.2.0.jar
+- vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.6.jar
+- vendor/jar-dependencies/runtime-jars/slf4j-log4j12-1.7.13.jar
+- vendor/jar-dependencies/runtime-jars/snappy-java-1.1.1.7.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -130,19 +136,19 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements:
 - jar 'org.apache.kafka:kafka-clients', '0.9.0.1'
 - jar 'org.slf4j:slf4j-log4j12', '1.7.13'
 rubyforge_project:
-rubygems_version: 2.4.8
+rubygems_version: 2.4.5
 signing_key:
 specification_version: 4
 summary: This input will read events from a Kafka topic. It uses the high level consumer API provided by Kafka to read messages from the broker