logstash-input-kafka 3.0.0.beta2 → 3.0.0.beta3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 22b41bd3a4cc464aa8fb16bad0a6d87cb29d9c2e
-  data.tar.gz: 3e6751e3ce4323a44b74f1234b5c69fcb89f841d
+  metadata.gz: f8d3e1a5d3ae1030b4c2401e92627d68079c764b
+  data.tar.gz: b5f497bf850e4b983ae618c060b1625ca489da76
 SHA512:
-  metadata.gz: e242a64f3b7e2f99e31c02c6cc5f7d9eecd781e1f36549ed61914176361f2f1ebac2a0827e83585733322613b6ec0d33da4799c3c056873aee675c5730c0f991
-  data.tar.gz: 9e8b29fa5c1412190e12ef6b0cefdf105b6e0c9aca494c54b66d1c51c98296bfaed6b91a47d38e5ac93bcccc6d2e810de1a979ac22fc03abce5bfbf76b15ebf4
+  metadata.gz: a6b759fe203404ca0cb633496fbd8e9ec17016836f06c1fa7a001445cfd7b6fe20e616368e1259c87739a94ac0f51f2a4462f26f4493e56f90132b1d35c80d35
+  data.tar.gz: 09049bef85ba6b173fbe6d55b2f807c54af1c3365cd0d56a562f6859470088bf3b162e7f19dbcf77b3c3c579541dcf08bc1a039aedef6f2099519b61f5008859

data/CHANGELOG.md

@@ -1,9 +1,12 @@
 # 3.0.0.beta2
  - Added SSL/TLS connection support to Kafka
+ - Breaking: Changed default codec to plain instead of SSL. Json codec is really slow when used 
+   with inputs because inputs by default are single threaded. This makes it a bad
+   first user experience. Plain codec is a much better default.
 
 # 3.0.0.beta1
  - Refactor to use new Java based consumer, bypassing jruby-kafka
- - Change configuration to match Kafka's configuration. This version is not backward compatible
+ - Breaking: Change configuration to match Kafka's configuration. This version is not backward compatible
 
 ## 2.0.0
  - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 

data/lib/logstash/inputs/kafka.rb

@@ -32,7 +32,7 @@ require 'logstash-input-kafka_jars.rb'
 class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   config_name 'kafka'
 
-  default :codec, 'json'
+  default :codec, 'plain'
 
   # The frequency in milliseconds that the consumer offsets are committed to Kafka.
   config :auto_commit_interval_ms, :validate => :string, :default => "10"
@@ -59,6 +59,9 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   config :client_id, :validate => :string, :default => "logstash"
   # Close idle connections after the number of milliseconds specified by this config.
   config :connections_max_idle_ms, :validate => :string
+  # Ideally you should have as many threads as the number of partitions for a perfect 
+  # balance — more threads than partitions means that some threads will be idle
+  config :consumer_threads, :validate => :number, :default => 1
   # If true, periodically commit to Kafka the offsets of messages already returned by the consumer. 
   # This committed offset will be used when the process fails as the position from 
   # which the consumption will begin.
@@ -111,9 +114,6 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   config :session_timeout_ms, :validate => :string, :default => "30000"
   # Java Class used to deserialize the record's value
   config :value_deserializer_class, :validate => :string, :default => "org.apache.kafka.common.serialization.StringDeserializer"
-  # Ideally you should have as many threads as the number of partitions for a perfect 
-  # balance — more threads than partitions means that some threads will be idle
-  config :consumer_threads, :validate => :number, :default => 1
   # A list of topics to subscribe to.
   config :topics, :validate => :array, :required => true
   # Time kafka consumer will wait to receive new messages from topics

data/logstash-input-kafka.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-kafka'
-  s.version         = '3.0.0.beta2'
+  s.version         = '3.0.0.beta3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = 'This input will read events from a Kafka topic. It uses the high level consumer API provided by Kafka to read messages from the broker'
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
   s.metadata = { 'logstash_plugin' => 'true', 'group' => 'input'}
 
   # Gem dependencies
-  s.add_runtime_dependency 'logstash-core', ">= 2.0.0.beta2", "< 3.0.0"
+  s.add_runtime_dependency 'logstash-core', ">= 2.0.0", "< 3.0.0"
   s.add_runtime_dependency 'logstash-codec-json'
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'stud', '>= 0.0.22', '< 0.1.0'

data/spec/integration/inputs/kafka_spec.rb

@@ -1,21 +1,40 @@
 # encoding: utf-8
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/inputs/kafka"
+require "digest"
 
 describe "input/kafka", :integration => true do
-  before do
-    props = java.util.Properties.new
-    props.put("bootstrap.servers", bootstrap_servers)
-    props.put("acks", "all")
-    props.put("retries", "0")
-    props.put("batch.size", "16384")
-    props.put("linger.ms", "1")
-    props.put("buffer.memory", "33554432")
-    props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer")
-    props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer")
-    producer = org.apache.kafka.clients.producer.KafkaProducer.new(props)
-    1000.times do |i|
-      producer.send(org.apache.kafka.clients.producer.ProducerRecord("test", i.to_s, i.to_s))
+  let(:partition3_config) { { 'topics' => ['topic3'], 'codec' => 'plain', 'auto_offset_reset' => 'earliest'} }
+  
+  let(:tries) { 60 }
+  let(:num_events) { 103 }
+  
+  def wait_until_count(queue)
+    num_tries = tries
+    while (num_tries > 0)
+      break if queue.size == num_events
+      num_tries -= 1
+      sleep 1
     end
   end
+  
+  def thread_it(kafka_input, queue)
+    Thread.new do
+      begin
+        kafka_input.run(queue)
+      end
+    end
+  end  
+    
+  it "should consume all messages from 3-partition topic" do
+    kafka_input = LogStash::Inputs::Kafka.new(partition3_config)
+    queue = Array.new
+    t = thread_it(kafka_input, queue)
+    t.run
+    
+    wait_until_count(queue)
+    
+    expect(queue.size).to eq(num_events)
+  end
+
 end

data/spec/unit/inputs/kafka_spec.rb

@@ -30,27 +30,10 @@ class MockConsumer
 end
 
 describe LogStash::Inputs::Kafka do
-  let(:config) { { 'topics' => ['test'], 'num_threads' => 4 } }
+  let(:config) { { 'topics' => ['test'], 'consumer_threads' => 4 } }
   subject { LogStash::Inputs::Kafka.new(config) }
 
   it "should register" do
     expect {subject.register}.to_not raise_error
   end
-
-  it "should run" do
-    expect(subject).to receive(:new_consumer) do
-      MockConsumer.new
-    end.exactly(4).times
-
-    subject.register
-    q = Queue.new
-    Thread.new do
-      while q.size < 13
-      end
-      subject.do_stop
-    end
-    subject.run(q)
-
-    expect(q.size).to eq(40)
-  end
 end

metadata

@@ -1,21 +1,21 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-kafka
 version: !ruby/object:Gem::Version
-  version: 3.0.0.beta2
+  version: 3.0.0.beta3
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-02-04 00:00:00.000000000 Z
+date: 2016-03-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 2.0.0.beta2
+        version: 2.0.0
     - - <
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 2.0.0.beta2
+        version: 2.0.0
     - - <
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -110,7 +110,7 @@ files:
 - logstash-input-kafka.gemspec
 - spec/integration/inputs/kafka_spec.rb
 - spec/unit/inputs/kafka_spec.rb
-- vendor/jar-dependencies/runtime-jars/kafka-clients-0.9.0.0.jar
+- vendor/jar-dependencies/runtime-jars/kafka-clients-0.9.0.1.jar
 - vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.13.jar
 - vendor/jar-dependencies/runtime-jars/slf4j-noop-1.7.13.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html