logstash-input-kafka 1.0.1 → 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +4 -4
- data/lib/logstash/inputs/kafka.rb +14 -16
- data/logstash-input-kafka.gemspec +4 -3
- data/spec/inputs/kafka_spec.rb +33 -4
- metadata +38 -26
- data/.gitignore +0 -3
- data/Rakefile +0 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 95340dea7b2fe3249a260ec1439d5dc986e50cfb
|
4
|
+
data.tar.gz: 02b71e6f503143eb3990981baa23f23fa0c8b679
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 41a9cde1503aec94f5d3b8d6ebdb798e9d42211338338b8dbb2014afb7a8b6b35ba7696f7d9d927e158839cc9a4b59219041991d8a46ca128351d5e7b4172a18
|
7
|
+
data.tar.gz: 51377efa0634c4561cbbfdec7c1f87e6b971232ff45e323a9c53f38bb4f354fa22e2e2846f4e668844ff3fe24ddca78b32b4a8b810825a9b79e385c2685a573d
|
data/README.md
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
# Logstash Plugin
|
2
2
|
|
3
|
-
This is a plugin for [Logstash](https://github.com/
|
3
|
+
This is a plugin for [Logstash](https://github.com/elastic/logstash).
|
4
4
|
|
5
5
|
It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
|
6
6
|
|
7
7
|
## Documentation
|
8
8
|
|
9
|
-
Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.
|
9
|
+
Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.co/guide/en/logstash/current/).
|
10
10
|
|
11
11
|
- For formatting code or config example, you can use the asciidoc `[source,ruby]` directive
|
12
|
-
- For more asciidoc formatting tips, see the excellent reference here https://github.com/
|
12
|
+
- For more asciidoc formatting tips, see the excellent reference here https://github.com/elastic/docs#asciidoc-guide
|
13
13
|
|
14
14
|
## Need Help?
|
15
15
|
|
@@ -83,4 +83,4 @@ Programming is not a required skill. Whatever you've seen about open source and
|
|
83
83
|
|
84
84
|
It is more important to the community that you are able to contribute.
|
85
85
|
|
86
|
-
For more information about contributing, see the [CONTRIBUTING](https://github.com/
|
86
|
+
For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.
|
@@ -1,6 +1,7 @@
|
|
1
1
|
require 'logstash/namespace'
|
2
2
|
require 'logstash/inputs/base'
|
3
3
|
require 'jruby-kafka'
|
4
|
+
require 'stud/interval'
|
4
5
|
|
5
6
|
# This input will read events from a Kafka topic. It uses the high level consumer API provided
|
6
7
|
# by Kafka to read messages from the broker. It also maintains the state of what has been
|
@@ -90,8 +91,6 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
|
|
90
91
|
config :decoder_class, :validate => :string, :default => 'kafka.serializer.DefaultDecoder'
|
91
92
|
# The serializer class for keys (defaults to the same default as for messages)
|
92
93
|
config :key_decoder_class, :validate => :string, :default => 'kafka.serializer.DefaultDecoder'
|
93
|
-
# The frequency in ms that the consumer offsets are committed to zookeeper.
|
94
|
-
config :auto_commit_interval, :validate => :string, :default => '1000'
|
95
94
|
|
96
95
|
|
97
96
|
public
|
@@ -112,8 +111,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
|
|
112
111
|
:allow_topics => @white_list,
|
113
112
|
:filter_topics => @black_list,
|
114
113
|
:value_decoder_class => @decoder_class,
|
115
|
-
:key_decoder_class => @key_decoder_class
|
116
|
-
:auto_commit_interval => @auto_commit_interval
|
114
|
+
:key_decoder_class => @key_decoder_class
|
117
115
|
}
|
118
116
|
if @reset_beginning
|
119
117
|
options[:reset_beginning] = 'from-beginning'
|
@@ -136,31 +134,32 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
|
|
136
134
|
@logger.info('Running kafka', :group_id => @group_id, :topic_id => @topic_id, :zk_connect => @zk_connect)
|
137
135
|
begin
|
138
136
|
@consumer_group.run(@consumer_threads,@kafka_client_queue)
|
139
|
-
|
140
|
-
|
137
|
+
|
138
|
+
while !stop?
|
139
|
+
if !@kafka_client_queue.empty?
|
141
140
|
event = @kafka_client_queue.pop
|
142
141
|
queue_event(event, logstash_queue)
|
143
142
|
end
|
144
|
-
rescue LogStash::ShutdownSignal
|
145
|
-
@logger.info('Kafka got shutdown signal')
|
146
|
-
@consumer_group.shutdown
|
147
143
|
end
|
144
|
+
|
148
145
|
until @kafka_client_queue.empty?
|
149
146
|
queue_event(@kafka_client_queue.pop,logstash_queue)
|
150
147
|
end
|
148
|
+
|
151
149
|
@logger.info('Done running kafka input')
|
152
150
|
rescue => e
|
153
151
|
@logger.warn('kafka client threw exception, restarting',
|
154
152
|
:exception => e)
|
155
|
-
|
156
|
-
|
157
|
-
end
|
158
|
-
sleep(Float(@consumer_restart_sleep_ms) * 1 / 1000)
|
159
|
-
retry
|
153
|
+
Stud.stoppable_sleep(Float(@consumer_restart_sleep_ms) * 1 / 1000) { stop? }
|
154
|
+
retry if !stop?
|
160
155
|
end
|
161
|
-
finished
|
162
156
|
end # def run
|
163
157
|
|
158
|
+
public
|
159
|
+
def stop
|
160
|
+
@consumer_group.shutdown if @consumer_group.running?
|
161
|
+
end
|
162
|
+
|
164
163
|
private
|
165
164
|
def create_consumer_group(options)
|
166
165
|
Kafka::Group.new(options)
|
@@ -185,5 +184,4 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
|
|
185
184
|
:backtrace => e.backtrace)
|
186
185
|
end # begin
|
187
186
|
end # def queue_event
|
188
|
-
|
189
187
|
end #class LogStash::Inputs::Kafka
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-input-kafka'
|
4
|
-
s.version = '
|
4
|
+
s.version = '2.0.0'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = 'This input will read events from a Kafka topic. It uses the high level consumer API provided by Kafka to read messages from the broker'
|
7
7
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
|
|
11
11
|
s.require_paths = ['lib']
|
12
12
|
|
13
13
|
# Files
|
14
|
-
s.files =
|
14
|
+
s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
|
15
15
|
|
16
16
|
# Tests
|
17
17
|
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
@@ -20,9 +20,10 @@ Gem::Specification.new do |s|
|
|
20
20
|
s.metadata = { 'logstash_plugin' => 'true', 'group' => 'input'}
|
21
21
|
|
22
22
|
# Gem dependencies
|
23
|
-
s.add_runtime_dependency "logstash-core",
|
23
|
+
s.add_runtime_dependency "logstash-core", "~> 2.0.0.snapshot"
|
24
24
|
s.add_runtime_dependency 'logstash-codec-json'
|
25
25
|
s.add_runtime_dependency 'logstash-codec-plain'
|
26
|
+
s.add_runtime_dependency 'stud', '>= 0.0.22', '< 0.1.0'
|
26
27
|
|
27
28
|
s.add_runtime_dependency 'jruby-kafka', ['>= 1.2.0', '< 2.0.0']
|
28
29
|
|
data/spec/inputs/kafka_spec.rb
CHANGED
@@ -7,8 +7,7 @@ class LogStash::Inputs::TestKafka < LogStash::Inputs::Kafka
|
|
7
7
|
private
|
8
8
|
def queue_event(msg, output_queue)
|
9
9
|
super(msg, output_queue)
|
10
|
-
|
11
|
-
raise LogStash::ShutdownSignal
|
10
|
+
do_stop
|
12
11
|
end
|
13
12
|
end
|
14
13
|
|
@@ -30,7 +29,26 @@ class TestKafkaGroup < Kafka::Group
|
|
30
29
|
end
|
31
30
|
end
|
32
31
|
|
33
|
-
|
32
|
+
class LogStash::Inputs::TestInfiniteKafka < LogStash::Inputs::Kafka
|
33
|
+
private
|
34
|
+
def queue_event(msg, output_queue)
|
35
|
+
super(msg, output_queue)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
class TestInfiniteKafkaGroup < Kafka::Group
|
40
|
+
def run(a_num_threads, a_queue)
|
41
|
+
blah = TestMessageAndMetadata.new(@topic, 0, nil, 'Kafka message')
|
42
|
+
Thread.new do
|
43
|
+
while true
|
44
|
+
a_queue << blah
|
45
|
+
sleep 0.2
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
describe LogStash::Inputs::Kafka do
|
34
52
|
let (:kafka_config) {{'topic_id' => 'test'}}
|
35
53
|
let (:empty_config) {{}}
|
36
54
|
let (:bad_kafka_config) {{'topic_id' => 'test', 'white_list' => 'other_topic'}}
|
@@ -57,6 +75,18 @@ describe 'inputs/kafka' do
|
|
57
75
|
expect {input.register}.to raise_error
|
58
76
|
end
|
59
77
|
|
78
|
+
it_behaves_like "an interruptible input plugin" do
|
79
|
+
let(:config) { kafka_config }
|
80
|
+
let(:mock_kafka_plugin) { LogStash::Inputs::TestInfiniteKafka.new(config) }
|
81
|
+
|
82
|
+
before :each do
|
83
|
+
allow(LogStash::Inputs::Kafka).to receive(:new).and_return(mock_kafka_plugin)
|
84
|
+
expect(subject).to receive(:create_consumer_group) do |options|
|
85
|
+
TestInfiniteKafkaGroup.new(options)
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
60
90
|
it 'should populate kafka config with default values' do
|
61
91
|
kafka = LogStash::Inputs::TestKafka.new(kafka_config)
|
62
92
|
insist {kafka.zk_connect} == 'localhost:2181'
|
@@ -98,5 +128,4 @@ describe 'inputs/kafka' do
|
|
98
128
|
insist { e['kafka']['partition'] } == 0
|
99
129
|
insist { e['kafka']['key'] } == nil
|
100
130
|
end
|
101
|
-
|
102
131
|
end
|
metadata
CHANGED
@@ -1,73 +1,78 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elasticsearch
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-09-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
15
|
+
requirements:
|
16
|
+
- - ~>
|
17
|
+
- !ruby/object:Gem::Version
|
18
|
+
version: 2.0.0.snapshot
|
14
19
|
name: logstash-core
|
20
|
+
prerelease: false
|
21
|
+
type: :runtime
|
15
22
|
version_requirements: !ruby/object:Gem::Requirement
|
16
23
|
requirements:
|
17
|
-
- -
|
24
|
+
- - ~>
|
18
25
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
20
|
-
|
21
|
-
- !ruby/object:Gem::Version
|
22
|
-
version: 2.0.0
|
26
|
+
version: 2.0.0.snapshot
|
27
|
+
- !ruby/object:Gem::Dependency
|
23
28
|
requirement: !ruby/object:Gem::Requirement
|
24
29
|
requirements:
|
25
30
|
- - '>='
|
26
31
|
- !ruby/object:Gem::Version
|
27
|
-
version:
|
28
|
-
|
29
|
-
- !ruby/object:Gem::Version
|
30
|
-
version: 2.0.0
|
32
|
+
version: '0'
|
33
|
+
name: logstash-codec-json
|
31
34
|
prerelease: false
|
32
35
|
type: :runtime
|
33
|
-
- !ruby/object:Gem::Dependency
|
34
|
-
name: logstash-codec-json
|
35
36
|
version_requirements: !ruby/object:Gem::Requirement
|
36
37
|
requirements:
|
37
38
|
- - '>='
|
38
39
|
- !ruby/object:Gem::Version
|
39
40
|
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
40
42
|
requirement: !ruby/object:Gem::Requirement
|
41
43
|
requirements:
|
42
44
|
- - '>='
|
43
45
|
- !ruby/object:Gem::Version
|
44
46
|
version: '0'
|
47
|
+
name: logstash-codec-plain
|
45
48
|
prerelease: false
|
46
49
|
type: :runtime
|
47
|
-
- !ruby/object:Gem::Dependency
|
48
|
-
name: logstash-codec-plain
|
49
50
|
version_requirements: !ruby/object:Gem::Requirement
|
50
51
|
requirements:
|
51
52
|
- - '>='
|
52
53
|
- !ruby/object:Gem::Version
|
53
54
|
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
54
56
|
requirement: !ruby/object:Gem::Requirement
|
55
57
|
requirements:
|
56
58
|
- - '>='
|
57
59
|
- !ruby/object:Gem::Version
|
58
|
-
version:
|
60
|
+
version: 0.0.22
|
61
|
+
- - <
|
62
|
+
- !ruby/object:Gem::Version
|
63
|
+
version: 0.1.0
|
64
|
+
name: stud
|
59
65
|
prerelease: false
|
60
66
|
type: :runtime
|
61
|
-
- !ruby/object:Gem::Dependency
|
62
|
-
name: jruby-kafka
|
63
67
|
version_requirements: !ruby/object:Gem::Requirement
|
64
68
|
requirements:
|
65
69
|
- - '>='
|
66
70
|
- !ruby/object:Gem::Version
|
67
|
-
version:
|
71
|
+
version: 0.0.22
|
68
72
|
- - <
|
69
73
|
- !ruby/object:Gem::Version
|
70
|
-
version:
|
74
|
+
version: 0.1.0
|
75
|
+
- !ruby/object:Gem::Dependency
|
71
76
|
requirement: !ruby/object:Gem::Requirement
|
72
77
|
requirements:
|
73
78
|
- - '>='
|
@@ -76,29 +81,37 @@ dependencies:
|
|
76
81
|
- - <
|
77
82
|
- !ruby/object:Gem::Version
|
78
83
|
version: 2.0.0
|
84
|
+
name: jruby-kafka
|
79
85
|
prerelease: false
|
80
86
|
type: :runtime
|
81
|
-
- !ruby/object:Gem::Dependency
|
82
|
-
name: logstash-devutils
|
83
87
|
version_requirements: !ruby/object:Gem::Requirement
|
84
88
|
requirements:
|
85
89
|
- - '>='
|
86
90
|
- !ruby/object:Gem::Version
|
87
|
-
version:
|
91
|
+
version: 1.2.0
|
92
|
+
- - <
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
version: 2.0.0
|
95
|
+
- !ruby/object:Gem::Dependency
|
88
96
|
requirement: !ruby/object:Gem::Requirement
|
89
97
|
requirements:
|
90
98
|
- - '>='
|
91
99
|
- !ruby/object:Gem::Version
|
92
100
|
version: '0'
|
101
|
+
name: logstash-devutils
|
93
102
|
prerelease: false
|
94
103
|
type: :development
|
104
|
+
version_requirements: !ruby/object:Gem::Requirement
|
105
|
+
requirements:
|
106
|
+
- - '>='
|
107
|
+
- !ruby/object:Gem::Version
|
108
|
+
version: '0'
|
95
109
|
description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
|
96
110
|
email: info@elastic.co
|
97
111
|
executables: []
|
98
112
|
extensions: []
|
99
113
|
extra_rdoc_files: []
|
100
114
|
files:
|
101
|
-
- .gitignore
|
102
115
|
- CHANGELOG.md
|
103
116
|
- CONTRIBUTORS
|
104
117
|
- DEVELOPER.md
|
@@ -106,7 +119,6 @@ files:
|
|
106
119
|
- LICENSE
|
107
120
|
- NOTICE.TXT
|
108
121
|
- README.md
|
109
|
-
- Rakefile
|
110
122
|
- lib/logstash/inputs/kafka.rb
|
111
123
|
- logstash-input-kafka.gemspec
|
112
124
|
- spec/inputs/kafka_spec.rb
|
@@ -132,7 +144,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
132
144
|
version: '0'
|
133
145
|
requirements: []
|
134
146
|
rubyforge_project:
|
135
|
-
rubygems_version: 2.4.
|
147
|
+
rubygems_version: 2.4.8
|
136
148
|
signing_key:
|
137
149
|
specification_version: 4
|
138
150
|
summary: This input will read events from a Kafka topic. It uses the high level consumer API provided by Kafka to read messages from the broker
|
data/.gitignore
DELETED