logstash-input-kafka 3.0.0.beta4 → 3.0.0.beta5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6bf0dff64749f974cfa649f10033a2ef9642d24f
-  data.tar.gz: 4f80203ce5e0244706d5fbaa4cd4edcfaff97b26
+  metadata.gz: ede94bdee56512bb6ab54d8dc1f66dccd4b71d78
+  data.tar.gz: 7473063725d735c21955cb85b5dd42c0a602c802
 SHA512:
-  metadata.gz: 0fe320217c7de713222de88f83aee68e87406e4dc5a5bed4a097db6647471247c62e88dbacbccaf786fad0f6c5e7fd32a9a42570b0a02e21e57536b91b0e465f
-  data.tar.gz: ac4692dcc79842c6cf7c1c010f972577d631a237951f3c3baf8190de4f0769adf405c29a63c2ec993b74767bea92bb589e08cc5e8215cf18a24dae0fe47d9c54
+  metadata.gz: fb255496bee1cd0ec990852b65f10f992b3dab7fa232bd1e721cc2ea09dce1ebcb903045017a22f8afb68807d2445bbe66215d41db492667a95afa455081ac86
+  data.tar.gz: 08cb1d08a5da9087513bbf394ad6ed47a673da957fff20cc2ac013eb783bac3d776d388cc5105dd66657be00d8db2f84e9fe924803fbd7aee0d2ebc2eb40bcfa

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+# 3.0.0.beta4
+ - Internal: Use jar dependency
+ - Fixed issue with snappy compression
+
+# 3.0.0.beta3 and 3.0.0.beta4
+ - Internal: Update gemspec dependency
+
+# 3.0.0.beta2
+ - internal: Use jar dependencies library instead of manually downloading jars
+ - Fixes "java.lang.ClassNotFoundException: org.xerial.snappy.SnappyOutputStream" issue (#50)
+
 # 3.0.0.beta2
  - Added SSL/TLS connection support to Kafka
  - Breaking: Changed default codec to plain instead of SSL. Json codec is really slow when used 

data/lib/logstash/inputs/kafka.rb

@@ -157,6 +157,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
           records = consumer.poll(poll_timeout_ms);
           for record in records do
             @codec.decode(record.value.to_s) do |event|
+              decorate(event)
               logstash_queue << event
             end
           end

data/logstash-input-kafka.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-kafka'
-  s.version         = '3.0.0.beta4'
+  s.version         = '3.0.0.beta5'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = 'This input will read events from a Kafka topic. It uses the high level consumer API provided by Kafka to read messages from the broker'
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -19,8 +19,13 @@ Gem::Specification.new do |s|
   # Special flag to let us know this is actually a logstash plugin
   s.metadata = { 'logstash_plugin' => 'true', 'group' => 'input'}
 
+  s.requirements << "jar 'org.apache.kafka:kafka-clients', '0.9.0.1'"
+  s.requirements << "jar 'org.slf4j:slf4j-log4j12', '1.7.13'"
+
+  s.add_development_dependency 'jar-dependencies', '~> 0.3.2'
+
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
+  s.add_runtime_dependency 'logstash-core', ">= 2.0.0", "< 3.0.0"
   s.add_runtime_dependency 'logstash-codec-json'
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'stud', '>= 0.0.22', '< 0.1.0'

data/spec/integration/inputs/kafka_spec.rb

@@ -5,25 +5,29 @@ require "digest"
 
 describe "input/kafka", :integration => true do
   let(:partition3_config) { { 'topics' => ['topic3'], 'codec' => 'plain', 'auto_offset_reset' => 'earliest'} }
+  let(:snappy_config) { { 'topics' => ['snappy_topic'], 'codec' => 'plain', 'auto_offset_reset' => 'earliest'} }
+  let(:lz4_config) { { 'topics' => ['lz4_topic'], 'codec' => 'plain', 'auto_offset_reset' => 'earliest'} }
   
   let(:tries) { 60 }
   let(:num_events) { 103 }
   
-  def wait_until_count(queue)
-    num_tries = tries
-    while (num_tries > 0)
-      break if queue.size == num_events
-      num_tries -= 1
-      sleep 1
-    end
-  end
-  
   def thread_it(kafka_input, queue)
     Thread.new do
       begin
         kafka_input.run(queue)
       end
     end
+  end
+  
+  def wait_for_events(queue, num_events)
+    begin
+      timeout(30) do
+        until queue.length == num_events do
+          sleep 1
+          next
+        end
+      end
+    end
   end  
     
   it "should consume all messages from 3-partition topic" do
@@ -31,10 +35,26 @@ describe "input/kafka", :integration => true do
     queue = Array.new
     t = thread_it(kafka_input, queue)
     t.run
-    
-    wait_until_count(queue)
-    
+    wait_for_events(queue, num_events)
+    expect(queue.size).to eq(num_events)
+  end
+  
+  it "should consume all messages from snappy 3-partition topic" do
+    kafka_input = LogStash::Inputs::Kafka.new(snappy_config)
+    queue = Array.new
+    t = thread_it(kafka_input, queue)
+    t.run
+    wait_for_events(queue, num_events)
     expect(queue.size).to eq(num_events)
   end
 
+  it "should consume all messages from lz4 3-partition topic" do
+    kafka_input = LogStash::Inputs::Kafka.new(lz4_config)
+    queue = Array.new
+    t = thread_it(kafka_input, queue)
+    t.run
+    wait_for_events(queue, num_events)
+    expect(queue.size).to eq(num_events)
+  end
+  
 end

metadata

@@ -1,29 +1,49 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-kafka
 version: !ruby/object:Gem::Version
-  version: 3.0.0.beta4
+  version: 3.0.0.beta5
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-31 00:00:00.000000000 Z
+date: 2016-04-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.0'
-  name: logstash-core-plugin-api
+        version: 0.3.2
+  name: jar-dependencies
   prerelease: false
-  type: :runtime
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: 0.3.2
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  name: logstash-core
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 3.0.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -105,8 +125,11 @@ files:
 - spec/integration/inputs/kafka_spec.rb
 - spec/unit/inputs/kafka_spec.rb
 - vendor/jar-dependencies/runtime-jars/kafka-clients-0.9.0.1.jar
-- vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.13.jar
-- vendor/jar-dependencies/runtime-jars/slf4j-noop-1.7.13.jar
+- vendor/jar-dependencies/runtime-jars/log4j-1.2.17.jar
+- vendor/jar-dependencies/runtime-jars/lz4-1.2.0.jar
+- vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.6.jar
+- vendor/jar-dependencies/runtime-jars/slf4j-log4j12-1.7.13.jar
+- vendor/jar-dependencies/runtime-jars/snappy-java-1.1.1.7.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -127,7 +150,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   - - '>'
     - !ruby/object:Gem::Version
       version: 1.3.1
-requirements: []
+requirements:
+- jar 'org.apache.kafka:kafka-clients', '0.9.0.1'
+- jar 'org.slf4j:slf4j-log4j12', '1.7.13'
 rubyforge_project:
 rubygems_version: 2.4.5
 signing_key: