logstash-input-kafka 0.1.9 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/DEVELOPER.md +6 -3
- data/lib/logstash/inputs/kafka.rb +35 -21
- data/logstash-input-kafka.gemspec +2 -6
- data/spec/inputs/kafka_spec.rb +32 -1
- metadata +40 -36
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 276ddeae9feef8142105c787ed19827747e6d924
|
4
|
+
data.tar.gz: 80064509099da476a1b19c3d655fad96633ca7e8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: dd14bff5887a805b851523880ea76839057e4e01f08c6ca69dbca2d718c67b9426467104312d94825f247b2dd3a5efcfed021744c5a5fece98e7de77d8a0f572
|
7
|
+
data.tar.gz: c4bc212388522afdfdefb84ad0eaa3d91dee786d45abd492c9b724a1d8ec0fb06f6cbad2f5d0ee6ee7020a836697ef2d73c5fb62ceba0e3ad3efc04a179a1c85
|
data/DEVELOPER.md
CHANGED
@@ -14,10 +14,13 @@ See http://kafka.apache.org/documentation.html#consumerconfigs for details about
|
|
14
14
|
|
15
15
|
input {
|
16
16
|
kafka {
|
17
|
-
topic_id => ... # string (
|
17
|
+
topic_id => ... # string (optional), default: nil, The topic to consume messages from. Can be a java regular expression for whitelist of topics.
|
18
|
+
white_list => ... # string (optional), default: nil, Blacklist of topics to exclude from consumption.
|
19
|
+
black_list => ... # string (optional), default: nil, Whitelist of topics to include for consumption.
|
18
20
|
zk_connect => ... # string (optional), default: "localhost:2181", Specifies the ZooKeeper connection string in the form hostname:port
|
19
21
|
group_id => ... # string (optional), default: "logstash", A string that uniquely identifies the group of consumer processes
|
20
22
|
reset_beginning => ... # boolean (optional), default: false, Specify whether to jump to beginning of the queue when there is no initial offset in ZK
|
23
|
+
auto_offset_reset => ... # string (optional), one of [ "largest", "smallest"] default => 'largest', Where consumer should start if group does not already have an established offset or offset is invalid
|
21
24
|
consumer_threads => ... # number (optional), default: 1, Number of threads to read from the partitions
|
22
25
|
queue_size => ... # number (optional), default: 20, Internal Logstash queue size used to hold events in memory
|
23
26
|
rebalance_max_retries => ... # number (optional), default: 4
|
@@ -26,8 +29,8 @@ See http://kafka.apache.org/documentation.html#consumerconfigs for details about
|
|
26
29
|
consumer_restart_on_error => ... # boolean (optional), default: true
|
27
30
|
consumer_restart_sleep_ms => ... # number (optional), default: 0
|
28
31
|
decorate_events => ... # boolean (optional), default: false, Option to add Kafka metadata like topic, message size to the event
|
29
|
-
consumer_id => ... # string (optional) default: nil
|
30
|
-
fetch_message_max_bytes => ... # number (optional) default: 1048576
|
32
|
+
consumer_id => ... # string (optional), default: nil
|
33
|
+
fetch_message_max_bytes => ... # number (optional), default: 1048576
|
31
34
|
}
|
32
35
|
}
|
33
36
|
|
@@ -1,13 +1,13 @@
|
|
1
1
|
require 'logstash/namespace'
|
2
2
|
require 'logstash/inputs/base'
|
3
|
-
require '
|
3
|
+
require 'jruby-kafka'
|
4
4
|
|
5
5
|
# This input will read events from a Kafka topic. It uses the high level consumer API provided
|
6
6
|
# by Kafka to read messages from the broker. It also maintains the state of what has been
|
7
7
|
# consumed using Zookeeper. The default input codec is json
|
8
8
|
#
|
9
|
-
#
|
10
|
-
# running on localhost. All the broker information is read from Zookeeper state
|
9
|
+
# You must configure `topic_id`, `white_list` or `black_list`. By default it will connect to a
|
10
|
+
# Zookeeper running on localhost. All the broker information is read from Zookeeper state
|
11
11
|
#
|
12
12
|
# Ideally you should have as many threads as the number of partitions for a perfect balance --
|
13
13
|
# more threads than partitions means that some threads will be idle
|
@@ -36,16 +36,19 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
|
|
36
36
|
# the same consumer group.
|
37
37
|
config :group_id, :validate => :string, :default => 'logstash'
|
38
38
|
# The topic to consume messages from
|
39
|
-
config :topic_id, :validate => :string, :
|
40
|
-
#
|
41
|
-
|
42
|
-
#
|
43
|
-
|
44
|
-
#
|
45
|
-
#
|
46
|
-
#
|
47
|
-
# work and the consumer threads will rejoin the consumer group.
|
39
|
+
config :topic_id, :validate => :string, :default => nil
|
40
|
+
# Whitelist of topics to include for consumption.
|
41
|
+
config :white_list, :validate => :string, :default => nil
|
42
|
+
# Blacklist of topics to exclude from consumption.
|
43
|
+
config :black_list, :validate => :string, :default => nil
|
44
|
+
# Reset the consumer group to start at the earliest message present in the log by clearing any
|
45
|
+
# offsets for the group stored in Zookeeper. This is destructive! Must be used in conjunction
|
46
|
+
# with auto_offset_reset => 'smallest'
|
48
47
|
config :reset_beginning, :validate => :boolean, :default => false
|
48
|
+
# `smallest` or `largest` - (optional, default `largest`) If the consumer does not already
|
49
|
+
# have an established offset or offset is invalid, start with the earliest message present in the
|
50
|
+
# log (`smallest`) or after the last message in the log (`largest`).
|
51
|
+
config :auto_offset_reset, :validate => %w( largest smallest ), :default => 'largest'
|
49
52
|
# Number of threads to read from the partitions. Ideally you should have as many threads as the
|
50
53
|
# number of partitions for a perfect balance. More threads than partitions means that some
|
51
54
|
# threads will be idle. Less threads means a single thread could be consuming from more than
|
@@ -81,23 +84,30 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
|
|
81
84
|
public
|
82
85
|
def register
|
83
86
|
LogStash::Logger.setup_log4j(@logger)
|
84
|
-
require 'jruby-kafka'
|
85
|
-
|
86
87
|
options = {
|
87
88
|
:zk_connect => @zk_connect,
|
88
89
|
:group_id => @group_id,
|
89
90
|
:topic_id => @topic_id,
|
91
|
+
:auto_offset_reset => @auto_offset_reset,
|
90
92
|
:rebalance_max_retries => @rebalance_max_retries,
|
91
93
|
:rebalance_backoff_ms => @rebalance_backoff_ms,
|
92
94
|
:consumer_timeout_ms => @consumer_timeout_ms,
|
93
95
|
:consumer_restart_on_error => @consumer_restart_on_error,
|
94
96
|
:consumer_restart_sleep_ms => @consumer_restart_sleep_ms,
|
95
97
|
:consumer_id => @consumer_id,
|
96
|
-
:fetch_message_max_bytes => @fetch_message_max_bytes
|
98
|
+
:fetch_message_max_bytes => @fetch_message_max_bytes,
|
99
|
+
:allow_topics => @white_list,
|
100
|
+
:filter_topics => @black_list
|
97
101
|
}
|
98
102
|
if @reset_beginning
|
99
103
|
options[:reset_beginning] = 'from-beginning'
|
100
104
|
end # if :reset_beginning
|
105
|
+
topic_or_filter = [@topic_id, @white_list, @black_list].compact
|
106
|
+
if topic_or_filter.count == 0
|
107
|
+
raise LogStash::ConfigurationError, 'topic_id, white_list or black_list required.'
|
108
|
+
elsif topic_or_filter.count > 1
|
109
|
+
raise LogStash::ConfigurationError, 'Invalid combination of topic_id, white_list or black_list. Use only one.'
|
110
|
+
end
|
101
111
|
@kafka_client_queue = SizedQueue.new(@queue_size)
|
102
112
|
@consumer_group = create_consumer_group(options)
|
103
113
|
@logger.info('Registering kafka', :group_id => @group_id, :topic_id => @topic_id, :zk_connect => @zk_connect)
|
@@ -113,14 +123,14 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
|
|
113
123
|
begin
|
114
124
|
while true
|
115
125
|
event = @kafka_client_queue.pop
|
116
|
-
queue_event(
|
126
|
+
queue_event(event, logstash_queue)
|
117
127
|
end
|
118
128
|
rescue LogStash::ShutdownSignal
|
119
129
|
@logger.info('Kafka got shutdown signal')
|
120
130
|
@consumer_group.shutdown
|
121
131
|
end
|
122
132
|
until @kafka_client_queue.empty?
|
123
|
-
queue_event(
|
133
|
+
queue_event(@kafka_client_queue.pop,logstash_queue)
|
124
134
|
end
|
125
135
|
@logger.info('Done running kafka input')
|
126
136
|
rescue => e
|
@@ -141,17 +151,21 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
|
|
141
151
|
end
|
142
152
|
|
143
153
|
private
|
144
|
-
def queue_event(
|
154
|
+
def queue_event(message_and_metadata, output_queue)
|
145
155
|
begin
|
146
|
-
@codec.decode(
|
156
|
+
@codec.decode("#{message_and_metadata.message}") do |event|
|
147
157
|
decorate(event)
|
148
158
|
if @decorate_events
|
149
|
-
event['kafka'] = {'msg_size' =>
|
159
|
+
event['kafka'] = {'msg_size' => event['message'].bytesize,
|
160
|
+
'topic' => message_and_metadata.topic,
|
161
|
+
'consumer_group' => @group_id,
|
162
|
+
'partition' => message_and_metadata.partition,
|
163
|
+
'key' => message_and_metadata.key}
|
150
164
|
end
|
151
165
|
output_queue << event
|
152
166
|
end # @codec.decode
|
153
167
|
rescue => e # parse or event creation error
|
154
|
-
@logger.error('Failed to create event', :message =>
|
168
|
+
@logger.error('Failed to create event', :message => "#{message_and_metadata.message}", :exception => e,
|
155
169
|
:backtrace => e.backtrace)
|
156
170
|
end # begin
|
157
171
|
end # def queue_event
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-input-kafka'
|
4
|
-
s.version = '0.1.
|
4
|
+
s.version = '0.1.10'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = 'This input will read events from a Kafka topic. It uses the high level consumer API provided by Kafka to read messages from the broker'
|
7
7
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
@@ -19,10 +19,6 @@ Gem::Specification.new do |s|
|
|
19
19
|
# Special flag to let us know this is actually a logstash plugin
|
20
20
|
s.metadata = { 'logstash_plugin' => 'true', 'group' => 'input'}
|
21
21
|
|
22
|
-
# Jar dependencies
|
23
|
-
s.requirements << "jar 'org.apache.kafka:kafka_2.9.2', '0.8.1.1'"
|
24
|
-
s.requirements << "jar 'org.slf4j:slf4j-log4j12', '1.7.10'"
|
25
|
-
|
26
22
|
# Gem dependencies
|
27
23
|
s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
|
28
24
|
s.add_runtime_dependency 'logstash-codec-json'
|
@@ -39,7 +35,7 @@ Gem::Specification.new do |s|
|
|
39
35
|
s.add_runtime_dependency 'ruby-maven', '3.1.1.0.8'
|
40
36
|
s.add_runtime_dependency "maven-tools", '1.0.7'
|
41
37
|
|
42
|
-
s.add_runtime_dependency 'jruby-kafka', ['>=0
|
38
|
+
s.add_runtime_dependency 'jruby-kafka', ['>= 1.1.0', '< 2.0.0']
|
43
39
|
|
44
40
|
s.add_development_dependency 'logstash-devutils'
|
45
41
|
end
|
data/spec/inputs/kafka_spec.rb
CHANGED
@@ -12,15 +12,29 @@ class LogStash::Inputs::TestKafka < LogStash::Inputs::Kafka
|
|
12
12
|
end
|
13
13
|
end
|
14
14
|
|
15
|
+
class TestMessageAndMetadata
|
16
|
+
attr_reader :topic, :partition, :key, :message
|
17
|
+
def initialize(topic, partition, key, message)
|
18
|
+
@topic = topic
|
19
|
+
@partition = partition
|
20
|
+
@key = key
|
21
|
+
@message = message
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
15
25
|
|
16
26
|
class TestKafkaGroup < Kafka::Group
|
17
27
|
def run(a_num_threads, a_queue)
|
18
|
-
|
28
|
+
blah = TestMessageAndMetadata.new(@topic, 0, nil, 'Kafka message')
|
29
|
+
a_queue << blah
|
19
30
|
end
|
20
31
|
end
|
21
32
|
|
22
33
|
describe 'inputs/kafka' do
|
23
34
|
let (:kafka_config) {{'topic_id' => 'test'}}
|
35
|
+
let (:empty_config) {{}}
|
36
|
+
let (:bad_kafka_config) {{'topic_id' => 'test', 'white_list' => 'other_topic'}}
|
37
|
+
let (:white_list_kafka_config) {{'white_list' => 'other_topic'}}
|
24
38
|
let (:decorated_kafka_config) {{'topic_id' => 'test', 'decorate_events' => true}}
|
25
39
|
|
26
40
|
it "should register" do
|
@@ -28,6 +42,21 @@ describe 'inputs/kafka' do
|
|
28
42
|
expect {input.register}.to_not raise_error
|
29
43
|
end
|
30
44
|
|
45
|
+
it "should register with whitelist" do
|
46
|
+
input = LogStash::Plugin.lookup("input", "kafka").new(white_list_kafka_config)
|
47
|
+
expect {input.register}.to_not raise_error
|
48
|
+
end
|
49
|
+
|
50
|
+
it "should fail with multiple topic configs" do
|
51
|
+
input = LogStash::Plugin.lookup("input", "kafka").new(empty_config)
|
52
|
+
expect {input.register}.to raise_error
|
53
|
+
end
|
54
|
+
|
55
|
+
it "should fail without topic configs" do
|
56
|
+
input = LogStash::Plugin.lookup("input", "kafka").new(bad_kafka_config)
|
57
|
+
expect {input.register}.to raise_error
|
58
|
+
end
|
59
|
+
|
31
60
|
it 'should populate kafka config with default values' do
|
32
61
|
kafka = LogStash::Inputs::TestKafka.new(kafka_config)
|
33
62
|
insist {kafka.zk_connect} == 'localhost:2181'
|
@@ -66,6 +95,8 @@ describe 'inputs/kafka' do
|
|
66
95
|
insist { e['kafka']['topic'] } == 'test'
|
67
96
|
insist { e['kafka']['consumer_group'] } == 'logstash'
|
68
97
|
insist { e['kafka']['msg_size'] } == 13
|
98
|
+
insist { e['kafka']['partition'] } == 0
|
99
|
+
insist { e['kafka']['key'] } == nil
|
69
100
|
end
|
70
101
|
|
71
102
|
end
|
metadata
CHANGED
@@ -1,17 +1,18 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.10
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elasticsearch
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-02-
|
11
|
+
date: 2015-02-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
|
14
|
+
name: logstash
|
15
|
+
version_requirements: !ruby/object:Gem::Requirement
|
15
16
|
requirements:
|
16
17
|
- - '>='
|
17
18
|
- !ruby/object:Gem::Version
|
@@ -19,10 +20,7 @@ dependencies:
|
|
19
20
|
- - <
|
20
21
|
- !ruby/object:Gem::Version
|
21
22
|
version: 2.0.0
|
22
|
-
|
23
|
-
prerelease: false
|
24
|
-
type: :runtime
|
25
|
-
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirement: !ruby/object:Gem::Requirement
|
26
24
|
requirements:
|
27
25
|
- - '>='
|
28
26
|
- !ruby/object:Gem::Version
|
@@ -30,104 +28,112 @@ dependencies:
|
|
30
28
|
- - <
|
31
29
|
- !ruby/object:Gem::Version
|
32
30
|
version: 2.0.0
|
31
|
+
prerelease: false
|
32
|
+
type: :runtime
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
|
+
name: logstash-codec-json
|
35
|
+
version_requirements: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - '>='
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: '0'
|
34
40
|
requirement: !ruby/object:Gem::Requirement
|
35
41
|
requirements:
|
36
42
|
- - '>='
|
37
43
|
- !ruby/object:Gem::Version
|
38
44
|
version: '0'
|
39
|
-
name: logstash-codec-json
|
40
45
|
prerelease: false
|
41
46
|
type: :runtime
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: logstash-codec-plain
|
42
49
|
version_requirements: !ruby/object:Gem::Requirement
|
43
50
|
requirements:
|
44
51
|
- - '>='
|
45
52
|
- !ruby/object:Gem::Version
|
46
53
|
version: '0'
|
47
|
-
- !ruby/object:Gem::Dependency
|
48
54
|
requirement: !ruby/object:Gem::Requirement
|
49
55
|
requirements:
|
50
56
|
- - '>='
|
51
57
|
- !ruby/object:Gem::Version
|
52
58
|
version: '0'
|
53
|
-
name: logstash-codec-plain
|
54
59
|
prerelease: false
|
55
60
|
type: :runtime
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
name: jar-dependencies
|
56
63
|
version_requirements: !ruby/object:Gem::Requirement
|
57
64
|
requirements:
|
58
|
-
- - '
|
65
|
+
- - '='
|
59
66
|
- !ruby/object:Gem::Version
|
60
|
-
version:
|
61
|
-
- !ruby/object:Gem::Dependency
|
67
|
+
version: 0.1.7
|
62
68
|
requirement: !ruby/object:Gem::Requirement
|
63
69
|
requirements:
|
64
70
|
- - '='
|
65
71
|
- !ruby/object:Gem::Version
|
66
72
|
version: 0.1.7
|
67
|
-
name: jar-dependencies
|
68
73
|
prerelease: false
|
69
74
|
type: :runtime
|
75
|
+
- !ruby/object:Gem::Dependency
|
76
|
+
name: ruby-maven
|
70
77
|
version_requirements: !ruby/object:Gem::Requirement
|
71
78
|
requirements:
|
72
79
|
- - '='
|
73
80
|
- !ruby/object:Gem::Version
|
74
|
-
version:
|
75
|
-
- !ruby/object:Gem::Dependency
|
81
|
+
version: 3.1.1.0.8
|
76
82
|
requirement: !ruby/object:Gem::Requirement
|
77
83
|
requirements:
|
78
84
|
- - '='
|
79
85
|
- !ruby/object:Gem::Version
|
80
86
|
version: 3.1.1.0.8
|
81
|
-
name: ruby-maven
|
82
87
|
prerelease: false
|
83
88
|
type: :runtime
|
89
|
+
- !ruby/object:Gem::Dependency
|
90
|
+
name: maven-tools
|
84
91
|
version_requirements: !ruby/object:Gem::Requirement
|
85
92
|
requirements:
|
86
93
|
- - '='
|
87
94
|
- !ruby/object:Gem::Version
|
88
|
-
version:
|
89
|
-
- !ruby/object:Gem::Dependency
|
95
|
+
version: 1.0.7
|
90
96
|
requirement: !ruby/object:Gem::Requirement
|
91
97
|
requirements:
|
92
98
|
- - '='
|
93
99
|
- !ruby/object:Gem::Version
|
94
100
|
version: 1.0.7
|
95
|
-
name: maven-tools
|
96
101
|
prerelease: false
|
97
102
|
type: :runtime
|
103
|
+
- !ruby/object:Gem::Dependency
|
104
|
+
name: jruby-kafka
|
98
105
|
version_requirements: !ruby/object:Gem::Requirement
|
99
106
|
requirements:
|
100
|
-
- - '
|
107
|
+
- - '>='
|
101
108
|
- !ruby/object:Gem::Version
|
102
|
-
version: 1.0
|
103
|
-
-
|
109
|
+
version: 1.1.0
|
110
|
+
- - <
|
111
|
+
- !ruby/object:Gem::Version
|
112
|
+
version: 2.0.0
|
104
113
|
requirement: !ruby/object:Gem::Requirement
|
105
114
|
requirements:
|
106
115
|
- - '>='
|
107
116
|
- !ruby/object:Gem::Version
|
108
|
-
version:
|
109
|
-
|
117
|
+
version: 1.1.0
|
118
|
+
- - <
|
119
|
+
- !ruby/object:Gem::Version
|
120
|
+
version: 2.0.0
|
110
121
|
prerelease: false
|
111
122
|
type: :runtime
|
123
|
+
- !ruby/object:Gem::Dependency
|
124
|
+
name: logstash-devutils
|
112
125
|
version_requirements: !ruby/object:Gem::Requirement
|
113
126
|
requirements:
|
114
127
|
- - '>='
|
115
128
|
- !ruby/object:Gem::Version
|
116
|
-
version: 0
|
117
|
-
- !ruby/object:Gem::Dependency
|
129
|
+
version: '0'
|
118
130
|
requirement: !ruby/object:Gem::Requirement
|
119
131
|
requirements:
|
120
132
|
- - '>='
|
121
133
|
- !ruby/object:Gem::Version
|
122
134
|
version: '0'
|
123
|
-
name: logstash-devutils
|
124
135
|
prerelease: false
|
125
136
|
type: :development
|
126
|
-
version_requirements: !ruby/object:Gem::Requirement
|
127
|
-
requirements:
|
128
|
-
- - '>='
|
129
|
-
- !ruby/object:Gem::Version
|
130
|
-
version: '0'
|
131
137
|
description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
|
132
138
|
email: info@elasticsearch.com
|
133
139
|
executables: []
|
@@ -164,9 +170,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
164
170
|
- - '>='
|
165
171
|
- !ruby/object:Gem::Version
|
166
172
|
version: '0'
|
167
|
-
requirements:
|
168
|
-
- jar 'org.apache.kafka:kafka_2.9.2', '0.8.1.1'
|
169
|
-
- jar 'org.slf4j:slf4j-log4j12', '1.7.10'
|
173
|
+
requirements: []
|
170
174
|
rubyforge_project:
|
171
175
|
rubygems_version: 2.1.9
|
172
176
|
signing_key:
|