logstash-input-kafka 1.0.1 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8e35448ada1beb21b7e9ca1e5285b9acb90807c5
-  data.tar.gz: 49fbe646c9847c8c48474f77d65bf6b9b2a54bd8
+  metadata.gz: 95340dea7b2fe3249a260ec1439d5dc986e50cfb
+  data.tar.gz: 02b71e6f503143eb3990981baa23f23fa0c8b679
 SHA512:
-  metadata.gz: 27d1050fbdfec370c97960caf8e0140c43ed8bc46c61ee5a7b68ab3b9b402e9697d175aa54f0299c25f2ac1b23f77f34117e5c390178d1951290c85fac5b6786
-  data.tar.gz: 1cda6c07296865c902f39bf34e1d69a29e5f531f8e85a6b6604ce6fe1562123c3ded680bfe7a7ebd1307bdbc0de8cc725e41610d3c9902ab38499111ae64eec0
+  metadata.gz: 41a9cde1503aec94f5d3b8d6ebdb798e9d42211338338b8dbb2014afb7a8b6b35ba7696f7d9d927e158839cc9a4b59219041991d8a46ca128351d5e7b4172a18
+  data.tar.gz: 51377efa0634c4561cbbfdec7c1f87e6b971232ff45e323a9c53f38bb4f354fa22e2e2846f4e668844ff3fe24ddca78b32b4a8b810825a9b79e385c2685a573d

data/README.md

@@ -1,15 +1,15 @@
 # Logstash Plugin
 
-This is a plugin for [Logstash](https://github.com/elasticsearch/logstash).
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
 
 ## Documentation
 
-Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elasticsearch.org/guide/en/logstash/current/).
+Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.co/guide/en/logstash/current/).
 
 - For formatting code or config example, you can use the asciidoc `[source,ruby]` directive
-- For more asciidoc formatting tips, see the excellent reference here https://github.com/elasticsearch/docs#asciidoc-guide
+- For more asciidoc formatting tips, see the excellent reference here https://github.com/elastic/docs#asciidoc-guide
 
 ## Need Help?
 
@@ -83,4 +83,4 @@ Programming is not a required skill. Whatever you've seen about open source and
 
 It is more important to the community that you are able to contribute.
 
-For more information about contributing, see the [CONTRIBUTING](https://github.com/elasticsearch/logstash/blob/master/CONTRIBUTING.md) file.
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.

data/lib/logstash/inputs/kafka.rb

@@ -1,6 +1,7 @@
 require 'logstash/namespace'
 require 'logstash/inputs/base'
 require 'jruby-kafka'
+require 'stud/interval'
 
 # This input will read events from a Kafka topic. It uses the high level consumer API provided
 # by Kafka to read messages from the broker. It also maintains the state of what has been
@@ -90,8 +91,6 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   config :decoder_class, :validate => :string, :default => 'kafka.serializer.DefaultDecoder'
   # The serializer class for keys (defaults to the same default as for messages)
   config :key_decoder_class, :validate => :string, :default => 'kafka.serializer.DefaultDecoder'
-  # The frequency in ms that the consumer offsets are committed to zookeeper.
-  config :auto_commit_interval, :validate => :string, :default => '1000'
 
 
   public
@@ -112,8 +111,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
         :allow_topics => @white_list,
         :filter_topics => @black_list,
         :value_decoder_class => @decoder_class,
-        :key_decoder_class => @key_decoder_class,
-        :auto_commit_interval => @auto_commit_interval
+        :key_decoder_class => @key_decoder_class
     }
     if @reset_beginning
       options[:reset_beginning] = 'from-beginning'
@@ -136,31 +134,32 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
     @logger.info('Running kafka', :group_id => @group_id, :topic_id => @topic_id, :zk_connect => @zk_connect)
     begin
       @consumer_group.run(@consumer_threads,@kafka_client_queue)
-      begin
-        while true
+
+      while !stop?
+        if !@kafka_client_queue.empty?
           event = @kafka_client_queue.pop
           queue_event(event, logstash_queue)
         end
-      rescue LogStash::ShutdownSignal
-        @logger.info('Kafka got shutdown signal')
-        @consumer_group.shutdown
       end
+
       until @kafka_client_queue.empty?
         queue_event(@kafka_client_queue.pop,logstash_queue)
       end
+
       @logger.info('Done running kafka input')
     rescue => e
       @logger.warn('kafka client threw exception, restarting',
                    :exception => e)
-      if @consumer_group.running?
-        @consumer_group.shutdown
-      end
-      sleep(Float(@consumer_restart_sleep_ms) * 1 / 1000)
-      retry
+      Stud.stoppable_sleep(Float(@consumer_restart_sleep_ms) * 1 / 1000) { stop? }
+      retry if !stop?
     end
-    finished
   end # def run
 
+  public
+  def stop
+    @consumer_group.shutdown if @consumer_group.running?
+  end
+
   private
   def create_consumer_group(options)
     Kafka::Group.new(options)
@@ -185,5 +184,4 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
                     :backtrace => e.backtrace)
     end # begin
   end # def queue_event
-
 end #class LogStash::Inputs::Kafka

data/logstash-input-kafka.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-kafka'
-  s.version         = '1.0.1'
+  s.version         = '2.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = 'This input will read events from a Kafka topic. It uses the high level consumer API provided by Kafka to read messages from the broker'
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
   s.require_paths = ['lib']
 
   # Files
-  s.files = `git ls-files`.split($\)
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
@@ -20,9 +20,10 @@ Gem::Specification.new do |s|
   s.metadata = { 'logstash_plugin' => 'true', 'group' => 'input'}
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0'
+  s.add_runtime_dependency "logstash-core", "~> 2.0.0.snapshot"
   s.add_runtime_dependency 'logstash-codec-json'
   s.add_runtime_dependency 'logstash-codec-plain'
+  s.add_runtime_dependency 'stud', '>= 0.0.22', '< 0.1.0'
 
   s.add_runtime_dependency 'jruby-kafka', ['>= 1.2.0', '< 2.0.0']
 

data/spec/inputs/kafka_spec.rb

@@ -7,8 +7,7 @@ class LogStash::Inputs::TestKafka < LogStash::Inputs::Kafka
   private
   def queue_event(msg, output_queue)
     super(msg, output_queue)
-    # need to raise exception here to stop the infinite loop
-    raise LogStash::ShutdownSignal
+    do_stop
   end
 end
 
@@ -30,7 +29,26 @@ class TestKafkaGroup < Kafka::Group
   end
 end
 
-describe 'inputs/kafka' do
+class LogStash::Inputs::TestInfiniteKafka < LogStash::Inputs::Kafka
+  private
+  def queue_event(msg, output_queue)
+    super(msg, output_queue)
+  end
+end
+
+class TestInfiniteKafkaGroup < Kafka::Group
+  def run(a_num_threads, a_queue)
+    blah = TestMessageAndMetadata.new(@topic, 0, nil, 'Kafka message')
+    Thread.new do
+      while true
+        a_queue << blah
+        sleep 0.2
+      end
+    end
+  end
+end
+
+describe LogStash::Inputs::Kafka do
   let (:kafka_config) {{'topic_id' => 'test'}}
   let (:empty_config) {{}}
   let (:bad_kafka_config) {{'topic_id' => 'test', 'white_list' => 'other_topic'}}
@@ -57,6 +75,18 @@ describe 'inputs/kafka' do
     expect {input.register}.to raise_error
   end
 
+  it_behaves_like "an interruptible input plugin" do
+    let(:config) { kafka_config }
+    let(:mock_kafka_plugin) { LogStash::Inputs::TestInfiniteKafka.new(config) }
+
+    before :each do
+      allow(LogStash::Inputs::Kafka).to receive(:new).and_return(mock_kafka_plugin)
+      expect(subject).to receive(:create_consumer_group) do |options|
+        TestInfiniteKafkaGroup.new(options)
+      end
+    end
+  end
+
   it 'should populate kafka config with default values' do
     kafka = LogStash::Inputs::TestKafka.new(kafka_config)
     insist {kafka.zk_connect} == 'localhost:2181'
@@ -98,5 +128,4 @@ describe 'inputs/kafka' do
     insist { e['kafka']['partition'] } == 0
     insist { e['kafka']['key'] } == nil
   end
-
 end

metadata

@@ -1,73 +1,78 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-kafka
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 2.0.0
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-23 00:00:00.000000000 Z
+date: 2015-09-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.0.0.snapshot
   name: logstash-core
+  prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 1.4.0
-    - - <
-      - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 2.0.0.snapshot
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 1.4.0
-    - - <
-      - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: '0'
+  name: logstash-codec-json
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-codec-json
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-codec-plain
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-codec-plain
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.0.22
+    - - <
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  name: stud
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: jruby-kafka
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 1.2.0
+        version: 0.0.22
     - - <
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 0.1.0
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
@@ -76,29 +81,37 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 2.0.0
+  name: jruby-kafka
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.2.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-devutils
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
 - CHANGELOG.md
 - CONTRIBUTORS
 - DEVELOPER.md
@@ -106,7 +119,6 @@ files:
 - LICENSE
 - NOTICE.TXT
 - README.md
-- Rakefile
 - lib/logstash/inputs/kafka.rb
 - logstash-input-kafka.gemspec
 - spec/inputs/kafka_spec.rb
@@ -132,7 +144,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.5
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: This input will read events from a Kafka topic. It uses the high level consumer API provided by Kafka to read messages from the broker

data/.gitignore

@@ -1,3 +0,0 @@
-*.gem
-Gemfile.lock
-.bundle

data/Rakefile

@@ -1,7 +0,0 @@
-@files=[]
-
-task :default do
-  system('rake -T')
-end
-
-require "logstash/devutils/rake"