logstash-output-kafka 3.0.0.beta2 → 3.0.0.beta3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b76080c2f6518c545f5bd0d1aa7b57819e6efd78
-  data.tar.gz: b794c352eb5dc1cdf22ead5cd384753bf3a317e1
+  metadata.gz: d0a2f506e4251a16b5b07ca803be2a1dfcb16e93
+  data.tar.gz: 9118badcb43ff86720641bce4d0862b258845f97
 SHA512:
-  metadata.gz: ff2c67cdb4d4e61c34926c48f3821e54b29194d0e06d149759fdde35055af29e35684c4b8c74183751dc83a269325bac3f3a896bf5ffe47bed9b9ad093cf0396
-  data.tar.gz: 3cc397e6a45c762a239260f326b5b7aeb63fc02c85b10275850437596b99b4e178e50dd58002d36ba0f50a6f2280dd6bda5a2e6cbc68613ad7a720291f07ff31
+  metadata.gz: b4fb49488983d073fd6c76eb39acb690576be2347155257f4d5a891651594b45b022714038d0c8d56c03b522eb8acf6ce869996beef5ba4f01db9828fcc54b80
+  data.tar.gz: fa28f7477753741cc59999fecde699cb0fc11938db993fd1f69f2add77bd028e6d3c928b41549b1fe435d127c34c849659ae8e0cc122b2029cfe09250befbaea

data/CHANGELOG.md

@@ -1,7 +1,17 @@
-# 2.0.4
+## 3.0.0.beta3
+ - Use jar dependencies
+ - Fixed snappy compression issue
+
+## 3.0.0.beta2
+ - Internal: Update gemspec dependency
+
+## 2.0.4
   - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
-# 2.0.3
+  - [Internal] Pin jruby-kafka to v1.6 to match input
+  
+## 2.0.3
   - New dependency requirements for logstash-core for the 5.0 release
+
 ## 3.0.0.beta1
  - Note: breaking changes in this version, and not backward compatible with Kafka 0.8 broker. 
    Please read carefully before installing

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012–2015 Elasticsearch <http://www.elastic.co>
+Copyright (c) 2012–2016 Elasticsearch <http://www.elastic.co>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/README.md

@@ -1,7 +1,6 @@
 # Logstash Plugin
 
-[![Build
-Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-kafka-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-kafka-unit/)
+[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-output-kafka.svg)](https://travis-ci.org/logstash-plugins/logstash-output-kafka)
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
@@ -70,7 +69,12 @@ gem "logstash-output-kafka", :path => "/your/local/logstash-output-kafka"
 ```
 - Install plugin
 ```sh
+# Logstash 2.3 and higher
+bin/logstash-plugin install --no-verify
+
+# Prior to Logstash 2.3
 bin/plugin install --no-verify
+
 ```
 - Run Logstash with your plugin
 ```sh

data/lib/logstash/outputs/kafka.rb

@@ -54,7 +54,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   config :buffer_memory, :validate => :number, :default => 33554432
   # The compression type for all data generated by the producer.
   # The default is none (i.e. no compression). Valid values are none, gzip, or snappy.
-  config :compression_type, :validate => ["none", "gzip", "snappy"], :default => "none"
+  config :compression_type, :validate => ["none", "gzip", "snappy", "lz4"], :default => "none"
   # The id string to pass to the server when making requests.
   # The purpose of this is to be able to track the source of requests beyond just
   # ip/port by allowing a logical application name to be included with the request

data/logstash-output-kafka.gemspec

@@ -1,10 +1,10 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-kafka'
-  s.version         = '3.0.0.beta2'
+  s.version         = '3.0.0.beta3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = 'Output events to a Kafka topic. This uses the Kafka Producer API to write messages to a topic on the broker'
-  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors         = ['Elasticsearch']
   s.email           = 'info@elastic.co'
   s.homepage        = "http://www.elastic.co/guide/en/logstash/current/index.html"
@@ -19,6 +19,11 @@ Gem::Specification.new do |s|
   # Special flag to let us know this is actually a logstash plugin
   s.metadata = { 'logstash_plugin' => 'true', 'group' => 'output'}
 
+  s.requirements << "jar 'org.apache.kafka:kafka-clients', '0.9.0.1'"
+  s.requirements << "jar 'org.slf4j:slf4j-log4j12', '1.7.13'"
+
+  s.add_development_dependency 'jar-dependencies', '~> 0.3.2'
+
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
   s.add_runtime_dependency 'logstash-codec-plain'
@@ -26,4 +31,5 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'poseidon'
+  s.add_development_dependency 'snappy'
 end

data/spec/integration/outputs/kafka_spec.rb

@@ -83,6 +83,38 @@ describe "outputs/kafka", :integration => true do
     end
   end
 
+  context 'when using snappy compression' do
+    let(:test_topic) { 'snappy_topic' }
+    let!(:consumer) do
+      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
+                                      test_topic, 0, :earliest_offset)
+    end
+    subject do
+      consumer.fetch
+    end
+
+    before :each do
+      config = base_config.merge({"topic_id" => test_topic, "compression_type" => "snappy"})
+      load_kafka_data(config)
+    end
+
+    it 'should have data integrity' do
+      expect(subject.size).to eq(num_events)
+      subject.each do |m|
+        expect(m.value).to eq(event.to_s)
+      end
+    end
+  end
+
+  context 'when using LZ4 compression' do
+    let(:test_topic) { 'lz4_topic' }
+
+    before :each do
+      config = base_config.merge({"topic_id" => test_topic, "compression_type" => "lz4"})
+      load_kafka_data(config)
+    end
+  end
+
   context 'when using multi partition topic' do
     let(:num_events) { 10 }
     let(:test_topic) { 'topic3' }

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-kafka
 version: !ruby/object:Gem::Version
-  version: 3.0.0.beta2
+  version: 3.0.0.beta3
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-24 00:00:00.000000000 Z
+date: 2016-04-28 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.3.2
+  name: jar-dependencies
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.3.2
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -80,7 +94,21 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: snappy
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
@@ -99,8 +127,11 @@ files:
 - spec/integration/outputs/kafka_spec.rb
 - spec/unit/outputs/kafka_spec.rb
 - vendor/jar-dependencies/runtime-jars/kafka-clients-0.9.0.1.jar
-- vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.13.jar
-- vendor/jar-dependencies/runtime-jars/slf4j-noop-1.7.13.jar
+- vendor/jar-dependencies/runtime-jars/log4j-1.2.17.jar
+- vendor/jar-dependencies/runtime-jars/lz4-1.2.0.jar
+- vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.6.jar
+- vendor/jar-dependencies/runtime-jars/slf4j-log4j12-1.7.13.jar
+- vendor/jar-dependencies/runtime-jars/snappy-java-1.1.1.7.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -121,7 +152,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   - - '>'
     - !ruby/object:Gem::Version
       version: 1.3.1
-requirements: []
+requirements:
+- jar 'org.apache.kafka:kafka-clients', '0.9.0.1'
+- jar 'org.slf4j:slf4j-log4j12', '1.7.13'
 rubyforge_project:
 rubygems_version: 2.4.5
 signing_key: