logstash-output-kafka 3.0.0.beta2 → 3.0.0.beta3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +12 -2
- data/LICENSE +1 -1
- data/README.md +6 -2
- data/lib/logstash/outputs/kafka.rb +1 -1
- data/logstash-output-kafka.gemspec +8 -2
- data/spec/integration/outputs/kafka_spec.rb +32 -0
- data/vendor/jar-dependencies/runtime-jars/log4j-1.2.17.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/lz4-1.2.0.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.6.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/slf4j-log4j12-1.7.13.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/snappy-java-1.1.1.7.jar +0 -0
- metadata +39 -6
- data/vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.13.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/slf4j-noop-1.7.13.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d0a2f506e4251a16b5b07ca803be2a1dfcb16e93
|
4
|
+
data.tar.gz: 9118badcb43ff86720641bce4d0862b258845f97
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b4fb49488983d073fd6c76eb39acb690576be2347155257f4d5a891651594b45b022714038d0c8d56c03b522eb8acf6ce869996beef5ba4f01db9828fcc54b80
|
7
|
+
data.tar.gz: fa28f7477753741cc59999fecde699cb0fc11938db993fd1f69f2add77bd028e6d3c928b41549b1fe435d127c34c849659ae8e0cc122b2029cfe09250befbaea
|
data/CHANGELOG.md
CHANGED
@@ -1,7 +1,17 @@
|
|
1
|
-
|
1
|
+
## 3.0.0.beta3
|
2
|
+
- Use jar dependencies
|
3
|
+
- Fixed snappy compression issue
|
4
|
+
|
5
|
+
## 3.0.0.beta2
|
6
|
+
- Internal: Update gemspec dependency
|
7
|
+
|
8
|
+
## 2.0.4
|
2
9
|
- Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
|
3
|
-
|
10
|
+
- [Internal] Pin jruby-kafka to v1.6 to match input
|
11
|
+
|
12
|
+
## 2.0.3
|
4
13
|
- New dependency requirements for logstash-core for the 5.0 release
|
14
|
+
|
5
15
|
## 3.0.0.beta1
|
6
16
|
- Note: breaking changes in this version, and not backward compatible with Kafka 0.8 broker.
|
7
17
|
Please read carefully before installing
|
data/LICENSE
CHANGED
data/README.md
CHANGED
@@ -1,7 +1,6 @@
|
|
1
1
|
# Logstash Plugin
|
2
2
|
|
3
|
-
[](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-kafka-unit/)
|
3
|
+
[](https://travis-ci.org/logstash-plugins/logstash-output-kafka)
|
5
4
|
|
6
5
|
This is a plugin for [Logstash](https://github.com/elastic/logstash).
|
7
6
|
|
@@ -70,7 +69,12 @@ gem "logstash-output-kafka", :path => "/your/local/logstash-output-kafka"
|
|
70
69
|
```
|
71
70
|
- Install plugin
|
72
71
|
```sh
|
72
|
+
# Logstash 2.3 and higher
|
73
|
+
bin/logstash-plugin install --no-verify
|
74
|
+
|
75
|
+
# Prior to Logstash 2.3
|
73
76
|
bin/plugin install --no-verify
|
77
|
+
|
74
78
|
```
|
75
79
|
- Run Logstash with your plugin
|
76
80
|
```sh
|
@@ -54,7 +54,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
|
|
54
54
|
config :buffer_memory, :validate => :number, :default => 33554432
|
55
55
|
# The compression type for all data generated by the producer.
|
56
56
|
# The default is none (i.e. no compression). Valid values are none, gzip, or snappy.
|
57
|
-
config :compression_type, :validate => ["none", "gzip", "snappy"], :default => "none"
|
57
|
+
config :compression_type, :validate => ["none", "gzip", "snappy", "lz4"], :default => "none"
|
58
58
|
# The id string to pass to the server when making requests.
|
59
59
|
# The purpose of this is to be able to track the source of requests beyond just
|
60
60
|
# ip/port by allowing a logical application name to be included with the request
|
@@ -1,10 +1,10 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-output-kafka'
|
4
|
-
s.version = '3.0.0.
|
4
|
+
s.version = '3.0.0.beta3'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = 'Output events to a Kafka topic. This uses the Kafka Producer API to write messages to a topic on the broker'
|
7
|
-
s.description = "This gem is a
|
7
|
+
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
8
8
|
s.authors = ['Elasticsearch']
|
9
9
|
s.email = 'info@elastic.co'
|
10
10
|
s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
|
@@ -19,6 +19,11 @@ Gem::Specification.new do |s|
|
|
19
19
|
# Special flag to let us know this is actually a logstash plugin
|
20
20
|
s.metadata = { 'logstash_plugin' => 'true', 'group' => 'output'}
|
21
21
|
|
22
|
+
s.requirements << "jar 'org.apache.kafka:kafka-clients', '0.9.0.1'"
|
23
|
+
s.requirements << "jar 'org.slf4j:slf4j-log4j12', '1.7.13'"
|
24
|
+
|
25
|
+
s.add_development_dependency 'jar-dependencies', '~> 0.3.2'
|
26
|
+
|
22
27
|
# Gem dependencies
|
23
28
|
s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
|
24
29
|
s.add_runtime_dependency 'logstash-codec-plain'
|
@@ -26,4 +31,5 @@ Gem::Specification.new do |s|
|
|
26
31
|
|
27
32
|
s.add_development_dependency 'logstash-devutils'
|
28
33
|
s.add_development_dependency 'poseidon'
|
34
|
+
s.add_development_dependency 'snappy'
|
29
35
|
end
|
@@ -83,6 +83,38 @@ describe "outputs/kafka", :integration => true do
|
|
83
83
|
end
|
84
84
|
end
|
85
85
|
|
86
|
+
context 'when using snappy compression' do
|
87
|
+
let(:test_topic) { 'snappy_topic' }
|
88
|
+
let!(:consumer) do
|
89
|
+
Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
|
90
|
+
test_topic, 0, :earliest_offset)
|
91
|
+
end
|
92
|
+
subject do
|
93
|
+
consumer.fetch
|
94
|
+
end
|
95
|
+
|
96
|
+
before :each do
|
97
|
+
config = base_config.merge({"topic_id" => test_topic, "compression_type" => "snappy"})
|
98
|
+
load_kafka_data(config)
|
99
|
+
end
|
100
|
+
|
101
|
+
it 'should have data integrity' do
|
102
|
+
expect(subject.size).to eq(num_events)
|
103
|
+
subject.each do |m|
|
104
|
+
expect(m.value).to eq(event.to_s)
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
context 'when using LZ4 compression' do
|
110
|
+
let(:test_topic) { 'lz4_topic' }
|
111
|
+
|
112
|
+
before :each do
|
113
|
+
config = base_config.merge({"topic_id" => test_topic, "compression_type" => "lz4"})
|
114
|
+
load_kafka_data(config)
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
86
118
|
context 'when using multi partition topic' do
|
87
119
|
let(:num_events) { 10 }
|
88
120
|
let(:test_topic) { 'topic3' }
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
metadata
CHANGED
@@ -1,15 +1,29 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.0.0.
|
4
|
+
version: 3.0.0.beta3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elasticsearch
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-04-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
15
|
+
requirements:
|
16
|
+
- - ~>
|
17
|
+
- !ruby/object:Gem::Version
|
18
|
+
version: 0.3.2
|
19
|
+
name: jar-dependencies
|
20
|
+
prerelease: false
|
21
|
+
type: :development
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ~>
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 0.3.2
|
13
27
|
- !ruby/object:Gem::Dependency
|
14
28
|
requirement: !ruby/object:Gem::Requirement
|
15
29
|
requirements:
|
@@ -80,7 +94,21 @@ dependencies:
|
|
80
94
|
- - '>='
|
81
95
|
- !ruby/object:Gem::Version
|
82
96
|
version: '0'
|
83
|
-
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
requirement: !ruby/object:Gem::Requirement
|
99
|
+
requirements:
|
100
|
+
- - '>='
|
101
|
+
- !ruby/object:Gem::Version
|
102
|
+
version: '0'
|
103
|
+
name: snappy
|
104
|
+
prerelease: false
|
105
|
+
type: :development
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - '>='
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
111
|
+
description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
|
84
112
|
email: info@elastic.co
|
85
113
|
executables: []
|
86
114
|
extensions: []
|
@@ -99,8 +127,11 @@ files:
|
|
99
127
|
- spec/integration/outputs/kafka_spec.rb
|
100
128
|
- spec/unit/outputs/kafka_spec.rb
|
101
129
|
- vendor/jar-dependencies/runtime-jars/kafka-clients-0.9.0.1.jar
|
102
|
-
- vendor/jar-dependencies/runtime-jars/
|
103
|
-
- vendor/jar-dependencies/runtime-jars/
|
130
|
+
- vendor/jar-dependencies/runtime-jars/log4j-1.2.17.jar
|
131
|
+
- vendor/jar-dependencies/runtime-jars/lz4-1.2.0.jar
|
132
|
+
- vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.6.jar
|
133
|
+
- vendor/jar-dependencies/runtime-jars/slf4j-log4j12-1.7.13.jar
|
134
|
+
- vendor/jar-dependencies/runtime-jars/snappy-java-1.1.1.7.jar
|
104
135
|
homepage: http://www.elastic.co/guide/en/logstash/current/index.html
|
105
136
|
licenses:
|
106
137
|
- Apache License (2.0)
|
@@ -121,7 +152,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
121
152
|
- - '>'
|
122
153
|
- !ruby/object:Gem::Version
|
123
154
|
version: 1.3.1
|
124
|
-
requirements:
|
155
|
+
requirements:
|
156
|
+
- jar 'org.apache.kafka:kafka-clients', '0.9.0.1'
|
157
|
+
- jar 'org.slf4j:slf4j-log4j12', '1.7.13'
|
125
158
|
rubyforge_project:
|
126
159
|
rubygems_version: 2.4.5
|
127
160
|
signing_key:
|
Binary file
|
Binary file
|