logstash-input-kafka 3.0.0.beta1 → 3.0.0.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9c608112bad7e897363c4d3328b7979746f5825e
-  data.tar.gz: 67cc809f94c52b10bad640ab916601958824ed83
+  metadata.gz: 22b41bd3a4cc464aa8fb16bad0a6d87cb29d9c2e
+  data.tar.gz: 3e6751e3ce4323a44b74f1234b5c69fcb89f841d
 SHA512:
-  metadata.gz: 2df9dd8df82666322cbbced05a00cb930b489c640a121d57dbe947d2ecb9ba9f025c9457fba0ae085bbe0c4fb47c95cebca3727531da36b70df43d6d54001c20
-  data.tar.gz: b3c4d7f98f3a81932279bc157665862a71ae1f3f62e6d88f0fb8a56dac5f82cb3984fa00ae904496dad4eef1528e0b885f2ff56046e671f6293956d606c02998
+  metadata.gz: e242a64f3b7e2f99e31c02c6cc5f7d9eecd781e1f36549ed61914176361f2f1ebac2a0827e83585733322613b6ec0d33da4799c3c056873aee675c5730c0f991
+  data.tar.gz: 9e8b29fa5c1412190e12ef6b0cefdf105b6e0c9aca494c54b66d1c51c98296bfaed6b91a47d38e5ac93bcccc6d2e810de1a979ac22fc03abce5bfbf76b15ebf4

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+# 3.0.0.beta2
+ - Added SSL/TLS connection support to Kafka
+
+# 3.0.0.beta1
+ - Refactor to use new Java based consumer, bypassing jruby-kafka
+ - Change configuration to match Kafka's configuration. This version is not backward compatible
+
 ## 2.0.0
  - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895

data/lib/logstash/inputs/kafka.rb

@@ -6,10 +6,11 @@ require 'logstash-input-kafka_jars.rb'
 
 # This input will read events from a Kafka topic. It uses the the newly designed
 # 0.9 version of consumer API[https://cwiki.apache.org/confluence/display/KAFKA/Kafka+0.9+Consumer+Rewrite+Design] 
-# provided by Kafka to read messages from the broker. This consumer is backward compatible and can
-# be used with 0.8.x brokers. 
+# provided by Kafka to read messages from the broker.
 #
-# The Logstash consumer handles group management and uses the default Kafka offset management
+# NOTE: This consumer is not backward compatible with 0.8.x brokers and needs a 0.9 broker.
+#
+# The Logstash Kafka consumer handles group management and uses the default offset management
 # strategy using Kafka topics.
 #
 # Logstash instances by default form a single logical group to subscribe to Kafka topics
@@ -25,10 +26,13 @@ require 'logstash-input-kafka_jars.rb'
 #
 # Kafka consumer configuration: http://kafka.apache.org/documentation.html#consumerconfigs
 #
+# This version also adds support for SSL/TLS security connection to Kafka. By default SSL is 
+# disabled but can be turned on as needed.
+#
 class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   config_name 'kafka'
 
-  default :codec, 'plain'
+  default :codec, 'json'
 
   # The frequency in milliseconds that the consumer offsets are committed to Kafka.
   config :auto_commit_interval_ms, :validate => :string, :default => "10"
@@ -109,7 +113,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   config :value_deserializer_class, :validate => :string, :default => "org.apache.kafka.common.serialization.StringDeserializer"
   # Ideally you should have as many threads as the number of partitions for a perfect 
   # balance — more threads than partitions means that some threads will be idle
-  config :num_threads, :validate => :number, :default => 1
+  config :consumer_threads, :validate => :number, :default => 1
   # A list of topics to subscribe to.
   config :topics, :validate => :array, :required => true
   # Time kafka consumer will wait to receive new messages from topics
@@ -134,7 +138,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
 
   public
   def run(logstash_queue)
-    @runner_consumers = num_threads.times.map { || create_consumer }
+    @runner_consumers = consumer_threads.times.map { || create_consumer }
     @runner_threads = @runner_consumers.map { |consumer| thread_runner(logstash_queue, consumer) }
     @runner_threads.each { |t| t.join }
   end # def run

data/logstash-input-kafka.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-kafka'
-  s.version         = '3.0.0.beta1'
+  s.version         = '3.0.0.beta2'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = 'This input will read events from a Kafka topic. It uses the high level consumer API provided by Kafka to read messages from the broker'
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-kafka
 version: !ruby/object:Gem::Version
-  version: 3.0.0.beta1
+  version: 3.0.0.beta2
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-01-14 00:00:00.000000000 Z
+date: 2016-02-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement