logstash-input-kafka 2.0.3 → 2.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b3867a2da2e1bcef41af77b310020da1c2a8cc78
-  data.tar.gz: d43d020db142194bd90d112b61ae09a80a6e5f73
+  metadata.gz: 8cbf018ee79064fe87ed7e0a6ec383bee0d552f5
+  data.tar.gz: e536bdb136d069db76dd7ff890a91c3cbbb24902
 SHA512:
-  metadata.gz: 319106e8a1dca73d8d4fca0db83b1835061e65fa14f5d0e37e092b64356b17d32cec30d01b88c60a415496f34a950e7915aa17f97bac83ecdce0793540f8190e
-  data.tar.gz: 780f0dbed7c2662e6dbffe1320155efa75de10c5899f8eec5a33012750a6d28ecb136cf0677b8aa8afeb1d9918bfc27f97fc7e5ee441b1ba2db4e7235814404b
+  metadata.gz: 50e38271ea18c0ca7edb4d1d16f3be5025f4ec0d6b204bdf7663af0831a70ea50cdc8e8a5a700f721d2d4b6656ac70989b1bfb43198fdc1b14a97fba2af023ba
+  data.tar.gz: 7e4212059e99825fbc47ee842826727dc78a2c74459a5a89ba3a7cdb8e9d0583e5db0cf16c3e0ac1a5079df4ccd1dd32566cdc9c702a00b6d388bf36f199662c

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 2.0.4
+ - Fix safe shutdown while plugin waits on Kafka for new events
+ - Expose auto_commit_interval_ms to control offset commit frequency
+
 ## 2.0.3
  - Fix infinite loop when no new messages are found in Kafka
 

data/lib/logstash/inputs/kafka.rb

@@ -50,6 +50,8 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   # have an established offset or offset is invalid, start with the earliest message present in the
   # log (`smallest`) or after the last message in the log (`largest`).
   config :auto_offset_reset, :validate => %w( largest smallest ), :default => 'largest'
+  # The frequency in ms that the consumer offsets are committed to zookeeper.
+  config :auto_commit_interval_ms, :validate => :number, :default => 1000
   # Number of threads to read from the partitions. Ideally you should have as many threads as the
   # number of partitions for a perfect balance. More threads than partitions means that some
   # threads will be idle. Less threads means a single thread could be consuming from more than
@@ -92,6 +94,8 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   # The serializer class for keys (defaults to the same default as for messages)
   config :key_decoder_class, :validate => :string, :default => 'kafka.serializer.DefaultDecoder'
 
+  class KafkaShutdownEvent; end
+  KAFKA_SHUTDOWN_EVENT = KafkaShutdownEvent.new
 
   public
   def register
@@ -101,6 +105,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
         :group_id => @group_id,
         :topic_id => @topic_id,
         :auto_offset_reset => @auto_offset_reset,
+        :auto_commit_interval => @auto_commit_interval_ms,
         :rebalance_max_retries => @rebalance_max_retries,
         :rebalance_backoff_ms => @rebalance_backoff_ms,
         :consumer_timeout_ms => @consumer_timeout_ms,
@@ -137,6 +142,9 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
 
       while !stop?
         event = @kafka_client_queue.pop
+        if event == KAFKA_SHUTDOWN_EVENT
+          break
+        end
         queue_event(event, logstash_queue)
       end
 
@@ -155,6 +163,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
 
   public
   def stop
+    @kafka_client_queue.push(KAFKA_SHUTDOWN_EVENT)
     @consumer_group.shutdown if @consumer_group.running?
   end
 

data/logstash-input-kafka.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-kafka'
-  s.version         = '2.0.3'
+  s.version         = '2.0.4'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = 'This input will read events from a Kafka topic. It uses the high level consumer API provided by Kafka to read messages from the broker'
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -25,7 +25,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'stud', '>= 0.0.22', '< 0.1.0'
 
-  s.add_runtime_dependency 'jruby-kafka', ['>= 1.2.0', '< 2.0.0']
+  s.add_runtime_dependency 'jruby-kafka', '1.5.0'
 
   s.add_development_dependency 'logstash-devutils'
 end

data/spec/inputs/kafka_spec.rb

@@ -42,7 +42,7 @@ class TestInfiniteKafkaGroup < Kafka::Group
     Thread.new do
       while true
         a_queue << blah
-        sleep 0.2
+        sleep 10
       end
     end
   end

metadata

@@ -1,17 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-kafka
 version: !ruby/object:Gem::Version
-  version: 2.0.3
+  version: 2.0.4
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-12-14 00:00:00.000000000 Z
+date: 2016-01-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash-core
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -19,10 +20,7 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 3.0.0
-  name: logstash-core
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -30,46 +28,38 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 3.0.0
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  name: logstash-codec-json
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-json
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-plain
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.0.22
-    - - <
-      - !ruby/object:Gem::Version
-        version: 0.1.0
-  name: stud
+        version: '0'
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: stud
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
@@ -78,40 +68,44 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 0.1.0
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 1.2.0
+        version: 0.0.22
     - - <
       - !ruby/object:Gem::Version
-        version: 2.0.0
-  name: jruby-kafka
+        version: 0.1.0
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: jruby-kafka
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: 1.2.0
-    - - <
+        version: 1.5.0
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 1.5.0
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash-devutils
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  prerelease: false
+  type: :development
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
@@ -150,7 +144,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.5
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: This input will read events from a Kafka topic. It uses the high level consumer API provided by Kafka to read messages from the broker