logstash-output-kafka 3.0.0.beta2 → 3.0.0.beta3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: b76080c2f6518c545f5bd0d1aa7b57819e6efd78
4
- data.tar.gz: b794c352eb5dc1cdf22ead5cd384753bf3a317e1
3
+ metadata.gz: d0a2f506e4251a16b5b07ca803be2a1dfcb16e93
4
+ data.tar.gz: 9118badcb43ff86720641bce4d0862b258845f97
5
5
  SHA512:
6
- metadata.gz: ff2c67cdb4d4e61c34926c48f3821e54b29194d0e06d149759fdde35055af29e35684c4b8c74183751dc83a269325bac3f3a896bf5ffe47bed9b9ad093cf0396
7
- data.tar.gz: 3cc397e6a45c762a239260f326b5b7aeb63fc02c85b10275850437596b99b4e178e50dd58002d36ba0f50a6f2280dd6bda5a2e6cbc68613ad7a720291f07ff31
6
+ metadata.gz: b4fb49488983d073fd6c76eb39acb690576be2347155257f4d5a891651594b45b022714038d0c8d56c03b522eb8acf6ce869996beef5ba4f01db9828fcc54b80
7
+ data.tar.gz: fa28f7477753741cc59999fecde699cb0fc11938db993fd1f69f2add77bd028e6d3c928b41549b1fe435d127c34c849659ae8e0cc122b2029cfe09250befbaea
@@ -1,7 +1,17 @@
1
- # 2.0.4
1
+ ## 3.0.0.beta3
2
+ - Use jar dependencies
3
+ - Fixed snappy compression issue
4
+
5
+ ## 3.0.0.beta2
6
+ - Internal: Update gemspec dependency
7
+
8
+ ## 2.0.4
2
9
  - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
3
- # 2.0.3
10
+ - [Internal] Pin jruby-kafka to v1.6 to match input
11
+
12
+ ## 2.0.3
4
13
  - New dependency requirements for logstash-core for the 5.0 release
14
+
5
15
  ## 3.0.0.beta1
6
16
  - Note: breaking changes in this version, and not backward compatible with Kafka 0.8 broker.
7
17
  Please read carefully before installing
data/LICENSE CHANGED
@@ -1,4 +1,4 @@
1
- Copyright (c) 2012–2015 Elasticsearch <http://www.elastic.co>
1
+ Copyright (c) 2012–2016 Elasticsearch <http://www.elastic.co>
2
2
 
3
3
  Licensed under the Apache License, Version 2.0 (the "License");
4
4
  you may not use this file except in compliance with the License.
data/README.md CHANGED
@@ -1,7 +1,6 @@
1
1
  # Logstash Plugin
2
2
 
3
- [![Build
4
- Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-kafka-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-kafka-unit/)
3
+ [![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-output-kafka.svg)](https://travis-ci.org/logstash-plugins/logstash-output-kafka)
5
4
 
6
5
  This is a plugin for [Logstash](https://github.com/elastic/logstash).
7
6
 
@@ -70,7 +69,12 @@ gem "logstash-output-kafka", :path => "/your/local/logstash-output-kafka"
70
69
  ```
71
70
  - Install plugin
72
71
  ```sh
72
+ # Logstash 2.3 and higher
73
+ bin/logstash-plugin install --no-verify
74
+
75
+ # Prior to Logstash 2.3
73
76
  bin/plugin install --no-verify
77
+
74
78
  ```
75
79
  - Run Logstash with your plugin
76
80
  ```sh
@@ -54,7 +54,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
54
54
  config :buffer_memory, :validate => :number, :default => 33554432
55
55
  # The compression type for all data generated by the producer.
56
56
  # The default is none (i.e. no compression). Valid values are none, gzip, or snappy.
57
- config :compression_type, :validate => ["none", "gzip", "snappy"], :default => "none"
57
+ config :compression_type, :validate => ["none", "gzip", "snappy", "lz4"], :default => "none"
58
58
  # The id string to pass to the server when making requests.
59
59
  # The purpose of this is to be able to track the source of requests beyond just
60
60
  # ip/port by allowing a logical application name to be included with the request
@@ -1,10 +1,10 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-output-kafka'
4
- s.version = '3.0.0.beta2'
4
+ s.version = '3.0.0.beta3'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = 'Output events to a Kafka topic. This uses the Kafka Producer API to write messages to a topic on the broker'
7
- s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
7
+ s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
8
8
  s.authors = ['Elasticsearch']
9
9
  s.email = 'info@elastic.co'
10
10
  s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
@@ -19,6 +19,11 @@ Gem::Specification.new do |s|
19
19
  # Special flag to let us know this is actually a logstash plugin
20
20
  s.metadata = { 'logstash_plugin' => 'true', 'group' => 'output'}
21
21
 
22
+ s.requirements << "jar 'org.apache.kafka:kafka-clients', '0.9.0.1'"
23
+ s.requirements << "jar 'org.slf4j:slf4j-log4j12', '1.7.13'"
24
+
25
+ s.add_development_dependency 'jar-dependencies', '~> 0.3.2'
26
+
22
27
  # Gem dependencies
23
28
  s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
24
29
  s.add_runtime_dependency 'logstash-codec-plain'
@@ -26,4 +31,5 @@ Gem::Specification.new do |s|
26
31
 
27
32
  s.add_development_dependency 'logstash-devutils'
28
33
  s.add_development_dependency 'poseidon'
34
+ s.add_development_dependency 'snappy'
29
35
  end
@@ -83,6 +83,38 @@ describe "outputs/kafka", :integration => true do
83
83
  end
84
84
  end
85
85
 
86
+ context 'when using snappy compression' do
87
+ let(:test_topic) { 'snappy_topic' }
88
+ let!(:consumer) do
89
+ Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
90
+ test_topic, 0, :earliest_offset)
91
+ end
92
+ subject do
93
+ consumer.fetch
94
+ end
95
+
96
+ before :each do
97
+ config = base_config.merge({"topic_id" => test_topic, "compression_type" => "snappy"})
98
+ load_kafka_data(config)
99
+ end
100
+
101
+ it 'should have data integrity' do
102
+ expect(subject.size).to eq(num_events)
103
+ subject.each do |m|
104
+ expect(m.value).to eq(event.to_s)
105
+ end
106
+ end
107
+ end
108
+
109
+ context 'when using LZ4 compression' do
110
+ let(:test_topic) { 'lz4_topic' }
111
+
112
+ before :each do
113
+ config = base_config.merge({"topic_id" => test_topic, "compression_type" => "lz4"})
114
+ load_kafka_data(config)
115
+ end
116
+ end
117
+
86
118
  context 'when using multi partition topic' do
87
119
  let(:num_events) { 10 }
88
120
  let(:test_topic) { 'topic3' }
metadata CHANGED
@@ -1,15 +1,29 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-kafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.0.0.beta2
4
+ version: 3.0.0.beta3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elasticsearch
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-03-24 00:00:00.000000000 Z
11
+ date: 2016-04-28 00:00:00.000000000 Z
12
12
  dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ requirement: !ruby/object:Gem::Requirement
15
+ requirements:
16
+ - - ~>
17
+ - !ruby/object:Gem::Version
18
+ version: 0.3.2
19
+ name: jar-dependencies
20
+ prerelease: false
21
+ type: :development
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ~>
25
+ - !ruby/object:Gem::Version
26
+ version: 0.3.2
13
27
  - !ruby/object:Gem::Dependency
14
28
  requirement: !ruby/object:Gem::Requirement
15
29
  requirements:
@@ -80,7 +94,21 @@ dependencies:
80
94
  - - '>='
81
95
  - !ruby/object:Gem::Version
82
96
  version: '0'
83
- description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
97
+ - !ruby/object:Gem::Dependency
98
+ requirement: !ruby/object:Gem::Requirement
99
+ requirements:
100
+ - - '>='
101
+ - !ruby/object:Gem::Version
102
+ version: '0'
103
+ name: snappy
104
+ prerelease: false
105
+ type: :development
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - '>='
109
+ - !ruby/object:Gem::Version
110
+ version: '0'
111
+ description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
84
112
  email: info@elastic.co
85
113
  executables: []
86
114
  extensions: []
@@ -99,8 +127,11 @@ files:
99
127
  - spec/integration/outputs/kafka_spec.rb
100
128
  - spec/unit/outputs/kafka_spec.rb
101
129
  - vendor/jar-dependencies/runtime-jars/kafka-clients-0.9.0.1.jar
102
- - vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.13.jar
103
- - vendor/jar-dependencies/runtime-jars/slf4j-noop-1.7.13.jar
130
+ - vendor/jar-dependencies/runtime-jars/log4j-1.2.17.jar
131
+ - vendor/jar-dependencies/runtime-jars/lz4-1.2.0.jar
132
+ - vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.6.jar
133
+ - vendor/jar-dependencies/runtime-jars/slf4j-log4j12-1.7.13.jar
134
+ - vendor/jar-dependencies/runtime-jars/snappy-java-1.1.1.7.jar
104
135
  homepage: http://www.elastic.co/guide/en/logstash/current/index.html
105
136
  licenses:
106
137
  - Apache License (2.0)
@@ -121,7 +152,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
121
152
  - - '>'
122
153
  - !ruby/object:Gem::Version
123
154
  version: 1.3.1
124
- requirements: []
155
+ requirements:
156
+ - jar 'org.apache.kafka:kafka-clients', '0.9.0.1'
157
+ - jar 'org.slf4j:slf4j-log4j12', '1.7.13'
125
158
  rubyforge_project:
126
159
  rubygems_version: 2.4.5
127
160
  signing_key: