logstash-input-kafka 0.1.9 → 0.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 0cf869a56fae132e42e6a9733672292db9df822a
4
- data.tar.gz: f8e8e1fe70b33fbf5516131d1fb51a9d87ba3f66
3
+ metadata.gz: 276ddeae9feef8142105c787ed19827747e6d924
4
+ data.tar.gz: 80064509099da476a1b19c3d655fad96633ca7e8
5
5
  SHA512:
6
- metadata.gz: 25cf9ff2e77349460df96cd4f7c37aa75a72c5bbc10b5ae5dce0986ad38775357a08e3358c2af343d075ee7868a01abbfa7f3647a679e386173c800f6bbc2370
7
- data.tar.gz: 5021919e331ea96f3955615954740ebeb37230b6e333c7b481b646953a6861823dd754f51dd00b035a2e3c1307d822919356ede73bb32cc425f8d425131f029c
6
+ metadata.gz: dd14bff5887a805b851523880ea76839057e4e01f08c6ca69dbca2d718c67b9426467104312d94825f247b2dd3a5efcfed021744c5a5fece98e7de77d8a0f572
7
+ data.tar.gz: c4bc212388522afdfdefb84ad0eaa3d91dee786d45abd492c9b724a1d8ec0fb06f6cbad2f5d0ee6ee7020a836697ef2d73c5fb62ceba0e3ad3efc04a179a1c85
data/DEVELOPER.md CHANGED
@@ -14,10 +14,13 @@ See http://kafka.apache.org/documentation.html#consumerconfigs for details about
14
14
 
15
15
  input {
16
16
  kafka {
17
- topic_id => ... # string (required), The topic to consume messages from
17
+ topic_id => ... # string (optional), default: nil, The topic to consume messages from. Can be a java regular expression for whitelist of topics.
18
+ white_list => ... # string (optional), default: nil, Blacklist of topics to exclude from consumption.
19
+ black_list => ... # string (optional), default: nil, Whitelist of topics to include for consumption.
18
20
  zk_connect => ... # string (optional), default: "localhost:2181", Specifies the ZooKeeper connection string in the form hostname:port
19
21
  group_id => ... # string (optional), default: "logstash", A string that uniquely identifies the group of consumer processes
20
22
  reset_beginning => ... # boolean (optional), default: false, Specify whether to jump to beginning of the queue when there is no initial offset in ZK
23
+ auto_offset_reset => ... # string (optional), one of [ "largest", "smallest"] default => 'largest', Where consumer should start if group does not already have an established offset or offset is invalid
21
24
  consumer_threads => ... # number (optional), default: 1, Number of threads to read from the partitions
22
25
  queue_size => ... # number (optional), default: 20, Internal Logstash queue size used to hold events in memory
23
26
  rebalance_max_retries => ... # number (optional), default: 4
@@ -26,8 +29,8 @@ See http://kafka.apache.org/documentation.html#consumerconfigs for details about
26
29
  consumer_restart_on_error => ... # boolean (optional), default: true
27
30
  consumer_restart_sleep_ms => ... # number (optional), default: 0
28
31
  decorate_events => ... # boolean (optional), default: false, Option to add Kafka metadata like topic, message size to the event
29
- consumer_id => ... # string (optional) default: nil
30
- fetch_message_max_bytes => ... # number (optional) default: 1048576
32
+ consumer_id => ... # string (optional), default: nil
33
+ fetch_message_max_bytes => ... # number (optional), default: 1048576
31
34
  }
32
35
  }
33
36
 
@@ -1,13 +1,13 @@
1
1
  require 'logstash/namespace'
2
2
  require 'logstash/inputs/base'
3
- require 'logstash-input-kafka_jars'
3
+ require 'jruby-kafka'
4
4
 
5
5
  # This input will read events from a Kafka topic. It uses the high level consumer API provided
6
6
  # by Kafka to read messages from the broker. It also maintains the state of what has been
7
7
  # consumed using Zookeeper. The default input codec is json
8
8
  #
9
- # The only required configuration is the topic name. By default it will connect to a Zookeeper
10
- # running on localhost. All the broker information is read from Zookeeper state
9
+ # You must configure `topic_id`, `white_list` or `black_list`. By default it will connect to a
10
+ # Zookeeper running on localhost. All the broker information is read from Zookeeper state
11
11
  #
12
12
  # Ideally you should have as many threads as the number of partitions for a perfect balance --
13
13
  # more threads than partitions means that some threads will be idle
@@ -36,16 +36,19 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
36
36
  # the same consumer group.
37
37
  config :group_id, :validate => :string, :default => 'logstash'
38
38
  # The topic to consume messages from
39
- config :topic_id, :validate => :string, :required => true
40
- # Specify whether to jump to beginning of the queue when there is no initial offset in
41
- # ZooKeeper, or if an offset is out of range. If this is `false`, messages are consumed
42
- # from the latest offset
43
- #
44
- # If `reset_beginning` is true, the consumer will check ZooKeeper to see if any other group members
45
- # are present and active. If not, the consumer deletes any offset information in the ZooKeeper
46
- # and starts at the smallest offset. If other group members are present `reset_beginning` will not
47
- # work and the consumer threads will rejoin the consumer group.
39
+ config :topic_id, :validate => :string, :default => nil
40
+ # Whitelist of topics to include for consumption.
41
+ config :white_list, :validate => :string, :default => nil
42
+ # Blacklist of topics to exclude from consumption.
43
+ config :black_list, :validate => :string, :default => nil
44
+ # Reset the consumer group to start at the earliest message present in the log by clearing any
45
+ # offsets for the group stored in Zookeeper. This is destructive! Must be used in conjunction
46
+ # with auto_offset_reset => 'smallest'
48
47
  config :reset_beginning, :validate => :boolean, :default => false
48
+ # `smallest` or `largest` - (optional, default `largest`) If the consumer does not already
49
+ # have an established offset or offset is invalid, start with the earliest message present in the
50
+ # log (`smallest`) or after the last message in the log (`largest`).
51
+ config :auto_offset_reset, :validate => %w( largest smallest ), :default => 'largest'
49
52
  # Number of threads to read from the partitions. Ideally you should have as many threads as the
50
53
  # number of partitions for a perfect balance. More threads than partitions means that some
51
54
  # threads will be idle. Less threads means a single thread could be consuming from more than
@@ -81,23 +84,30 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
81
84
  public
82
85
  def register
83
86
  LogStash::Logger.setup_log4j(@logger)
84
- require 'jruby-kafka'
85
-
86
87
  options = {
87
88
  :zk_connect => @zk_connect,
88
89
  :group_id => @group_id,
89
90
  :topic_id => @topic_id,
91
+ :auto_offset_reset => @auto_offset_reset,
90
92
  :rebalance_max_retries => @rebalance_max_retries,
91
93
  :rebalance_backoff_ms => @rebalance_backoff_ms,
92
94
  :consumer_timeout_ms => @consumer_timeout_ms,
93
95
  :consumer_restart_on_error => @consumer_restart_on_error,
94
96
  :consumer_restart_sleep_ms => @consumer_restart_sleep_ms,
95
97
  :consumer_id => @consumer_id,
96
- :fetch_message_max_bytes => @fetch_message_max_bytes
98
+ :fetch_message_max_bytes => @fetch_message_max_bytes,
99
+ :allow_topics => @white_list,
100
+ :filter_topics => @black_list
97
101
  }
98
102
  if @reset_beginning
99
103
  options[:reset_beginning] = 'from-beginning'
100
104
  end # if :reset_beginning
105
+ topic_or_filter = [@topic_id, @white_list, @black_list].compact
106
+ if topic_or_filter.count == 0
107
+ raise LogStash::ConfigurationError, 'topic_id, white_list or black_list required.'
108
+ elsif topic_or_filter.count > 1
109
+ raise LogStash::ConfigurationError, 'Invalid combination of topic_id, white_list or black_list. Use only one.'
110
+ end
101
111
  @kafka_client_queue = SizedQueue.new(@queue_size)
102
112
  @consumer_group = create_consumer_group(options)
103
113
  @logger.info('Registering kafka', :group_id => @group_id, :topic_id => @topic_id, :zk_connect => @zk_connect)
@@ -113,14 +123,14 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
113
123
  begin
114
124
  while true
115
125
  event = @kafka_client_queue.pop
116
- queue_event("#{event}",logstash_queue)
126
+ queue_event(event, logstash_queue)
117
127
  end
118
128
  rescue LogStash::ShutdownSignal
119
129
  @logger.info('Kafka got shutdown signal')
120
130
  @consumer_group.shutdown
121
131
  end
122
132
  until @kafka_client_queue.empty?
123
- queue_event("#{@kafka_client_queue.pop}",logstash_queue)
133
+ queue_event(@kafka_client_queue.pop,logstash_queue)
124
134
  end
125
135
  @logger.info('Done running kafka input')
126
136
  rescue => e
@@ -141,17 +151,21 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
141
151
  end
142
152
 
143
153
  private
144
- def queue_event(msg, output_queue)
154
+ def queue_event(message_and_metadata, output_queue)
145
155
  begin
146
- @codec.decode(msg) do |event|
156
+ @codec.decode("#{message_and_metadata.message}") do |event|
147
157
  decorate(event)
148
158
  if @decorate_events
149
- event['kafka'] = {'msg_size' => msg.bytesize, 'topic' => @topic_id, 'consumer_group' => @group_id}
159
+ event['kafka'] = {'msg_size' => event['message'].bytesize,
160
+ 'topic' => message_and_metadata.topic,
161
+ 'consumer_group' => @group_id,
162
+ 'partition' => message_and_metadata.partition,
163
+ 'key' => message_and_metadata.key}
150
164
  end
151
165
  output_queue << event
152
166
  end # @codec.decode
153
167
  rescue => e # parse or event creation error
154
- @logger.error('Failed to create event', :message => msg, :exception => e,
168
+ @logger.error('Failed to create event', :message => "#{message_and_metadata.message}", :exception => e,
155
169
  :backtrace => e.backtrace)
156
170
  end # begin
157
171
  end # def queue_event
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-input-kafka'
4
- s.version = '0.1.9'
4
+ s.version = '0.1.10'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = 'This input will read events from a Kafka topic. It uses the high level consumer API provided by Kafka to read messages from the broker'
7
7
  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -19,10 +19,6 @@ Gem::Specification.new do |s|
19
19
  # Special flag to let us know this is actually a logstash plugin
20
20
  s.metadata = { 'logstash_plugin' => 'true', 'group' => 'input'}
21
21
 
22
- # Jar dependencies
23
- s.requirements << "jar 'org.apache.kafka:kafka_2.9.2', '0.8.1.1'"
24
- s.requirements << "jar 'org.slf4j:slf4j-log4j12', '1.7.10'"
25
-
26
22
  # Gem dependencies
27
23
  s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
28
24
  s.add_runtime_dependency 'logstash-codec-json'
@@ -39,7 +35,7 @@ Gem::Specification.new do |s|
39
35
  s.add_runtime_dependency 'ruby-maven', '3.1.1.0.8'
40
36
  s.add_runtime_dependency "maven-tools", '1.0.7'
41
37
 
42
- s.add_runtime_dependency 'jruby-kafka', ['>=0.2.1']
38
+ s.add_runtime_dependency 'jruby-kafka', ['>= 1.1.0', '< 2.0.0']
43
39
 
44
40
  s.add_development_dependency 'logstash-devutils'
45
41
  end
@@ -12,15 +12,29 @@ class LogStash::Inputs::TestKafka < LogStash::Inputs::Kafka
12
12
  end
13
13
  end
14
14
 
15
+ class TestMessageAndMetadata
16
+ attr_reader :topic, :partition, :key, :message
17
+ def initialize(topic, partition, key, message)
18
+ @topic = topic
19
+ @partition = partition
20
+ @key = key
21
+ @message = message
22
+ end
23
+ end
24
+
15
25
 
16
26
  class TestKafkaGroup < Kafka::Group
17
27
  def run(a_num_threads, a_queue)
18
- a_queue << 'Kafka message'
28
+ blah = TestMessageAndMetadata.new(@topic, 0, nil, 'Kafka message')
29
+ a_queue << blah
19
30
  end
20
31
  end
21
32
 
22
33
  describe 'inputs/kafka' do
23
34
  let (:kafka_config) {{'topic_id' => 'test'}}
35
+ let (:empty_config) {{}}
36
+ let (:bad_kafka_config) {{'topic_id' => 'test', 'white_list' => 'other_topic'}}
37
+ let (:white_list_kafka_config) {{'white_list' => 'other_topic'}}
24
38
  let (:decorated_kafka_config) {{'topic_id' => 'test', 'decorate_events' => true}}
25
39
 
26
40
  it "should register" do
@@ -28,6 +42,21 @@ describe 'inputs/kafka' do
28
42
  expect {input.register}.to_not raise_error
29
43
  end
30
44
 
45
+ it "should register with whitelist" do
46
+ input = LogStash::Plugin.lookup("input", "kafka").new(white_list_kafka_config)
47
+ expect {input.register}.to_not raise_error
48
+ end
49
+
50
+ it "should fail with multiple topic configs" do
51
+ input = LogStash::Plugin.lookup("input", "kafka").new(empty_config)
52
+ expect {input.register}.to raise_error
53
+ end
54
+
55
+ it "should fail without topic configs" do
56
+ input = LogStash::Plugin.lookup("input", "kafka").new(bad_kafka_config)
57
+ expect {input.register}.to raise_error
58
+ end
59
+
31
60
  it 'should populate kafka config with default values' do
32
61
  kafka = LogStash::Inputs::TestKafka.new(kafka_config)
33
62
  insist {kafka.zk_connect} == 'localhost:2181'
@@ -66,6 +95,8 @@ describe 'inputs/kafka' do
66
95
  insist { e['kafka']['topic'] } == 'test'
67
96
  insist { e['kafka']['consumer_group'] } == 'logstash'
68
97
  insist { e['kafka']['msg_size'] } == 13
98
+ insist { e['kafka']['partition'] } == 0
99
+ insist { e['kafka']['key'] } == nil
69
100
  end
70
101
 
71
102
  end
metadata CHANGED
@@ -1,17 +1,18 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-input-kafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.9
4
+ version: 0.1.10
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elasticsearch
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-02-13 00:00:00.000000000 Z
11
+ date: 2015-02-25 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- requirement: !ruby/object:Gem::Requirement
14
+ name: logstash
15
+ version_requirements: !ruby/object:Gem::Requirement
15
16
  requirements:
16
17
  - - '>='
17
18
  - !ruby/object:Gem::Version
@@ -19,10 +20,7 @@ dependencies:
19
20
  - - <
20
21
  - !ruby/object:Gem::Version
21
22
  version: 2.0.0
22
- name: logstash
23
- prerelease: false
24
- type: :runtime
25
- version_requirements: !ruby/object:Gem::Requirement
23
+ requirement: !ruby/object:Gem::Requirement
26
24
  requirements:
27
25
  - - '>='
28
26
  - !ruby/object:Gem::Version
@@ -30,104 +28,112 @@ dependencies:
30
28
  - - <
31
29
  - !ruby/object:Gem::Version
32
30
  version: 2.0.0
31
+ prerelease: false
32
+ type: :runtime
33
33
  - !ruby/object:Gem::Dependency
34
+ name: logstash-codec-json
35
+ version_requirements: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - '>='
38
+ - !ruby/object:Gem::Version
39
+ version: '0'
34
40
  requirement: !ruby/object:Gem::Requirement
35
41
  requirements:
36
42
  - - '>='
37
43
  - !ruby/object:Gem::Version
38
44
  version: '0'
39
- name: logstash-codec-json
40
45
  prerelease: false
41
46
  type: :runtime
47
+ - !ruby/object:Gem::Dependency
48
+ name: logstash-codec-plain
42
49
  version_requirements: !ruby/object:Gem::Requirement
43
50
  requirements:
44
51
  - - '>='
45
52
  - !ruby/object:Gem::Version
46
53
  version: '0'
47
- - !ruby/object:Gem::Dependency
48
54
  requirement: !ruby/object:Gem::Requirement
49
55
  requirements:
50
56
  - - '>='
51
57
  - !ruby/object:Gem::Version
52
58
  version: '0'
53
- name: logstash-codec-plain
54
59
  prerelease: false
55
60
  type: :runtime
61
+ - !ruby/object:Gem::Dependency
62
+ name: jar-dependencies
56
63
  version_requirements: !ruby/object:Gem::Requirement
57
64
  requirements:
58
- - - '>='
65
+ - - '='
59
66
  - !ruby/object:Gem::Version
60
- version: '0'
61
- - !ruby/object:Gem::Dependency
67
+ version: 0.1.7
62
68
  requirement: !ruby/object:Gem::Requirement
63
69
  requirements:
64
70
  - - '='
65
71
  - !ruby/object:Gem::Version
66
72
  version: 0.1.7
67
- name: jar-dependencies
68
73
  prerelease: false
69
74
  type: :runtime
75
+ - !ruby/object:Gem::Dependency
76
+ name: ruby-maven
70
77
  version_requirements: !ruby/object:Gem::Requirement
71
78
  requirements:
72
79
  - - '='
73
80
  - !ruby/object:Gem::Version
74
- version: 0.1.7
75
- - !ruby/object:Gem::Dependency
81
+ version: 3.1.1.0.8
76
82
  requirement: !ruby/object:Gem::Requirement
77
83
  requirements:
78
84
  - - '='
79
85
  - !ruby/object:Gem::Version
80
86
  version: 3.1.1.0.8
81
- name: ruby-maven
82
87
  prerelease: false
83
88
  type: :runtime
89
+ - !ruby/object:Gem::Dependency
90
+ name: maven-tools
84
91
  version_requirements: !ruby/object:Gem::Requirement
85
92
  requirements:
86
93
  - - '='
87
94
  - !ruby/object:Gem::Version
88
- version: 3.1.1.0.8
89
- - !ruby/object:Gem::Dependency
95
+ version: 1.0.7
90
96
  requirement: !ruby/object:Gem::Requirement
91
97
  requirements:
92
98
  - - '='
93
99
  - !ruby/object:Gem::Version
94
100
  version: 1.0.7
95
- name: maven-tools
96
101
  prerelease: false
97
102
  type: :runtime
103
+ - !ruby/object:Gem::Dependency
104
+ name: jruby-kafka
98
105
  version_requirements: !ruby/object:Gem::Requirement
99
106
  requirements:
100
- - - '='
107
+ - - '>='
101
108
  - !ruby/object:Gem::Version
102
- version: 1.0.7
103
- - !ruby/object:Gem::Dependency
109
+ version: 1.1.0
110
+ - - <
111
+ - !ruby/object:Gem::Version
112
+ version: 2.0.0
104
113
  requirement: !ruby/object:Gem::Requirement
105
114
  requirements:
106
115
  - - '>='
107
116
  - !ruby/object:Gem::Version
108
- version: 0.2.1
109
- name: jruby-kafka
117
+ version: 1.1.0
118
+ - - <
119
+ - !ruby/object:Gem::Version
120
+ version: 2.0.0
110
121
  prerelease: false
111
122
  type: :runtime
123
+ - !ruby/object:Gem::Dependency
124
+ name: logstash-devutils
112
125
  version_requirements: !ruby/object:Gem::Requirement
113
126
  requirements:
114
127
  - - '>='
115
128
  - !ruby/object:Gem::Version
116
- version: 0.2.1
117
- - !ruby/object:Gem::Dependency
129
+ version: '0'
118
130
  requirement: !ruby/object:Gem::Requirement
119
131
  requirements:
120
132
  - - '>='
121
133
  - !ruby/object:Gem::Version
122
134
  version: '0'
123
- name: logstash-devutils
124
135
  prerelease: false
125
136
  type: :development
126
- version_requirements: !ruby/object:Gem::Requirement
127
- requirements:
128
- - - '>='
129
- - !ruby/object:Gem::Version
130
- version: '0'
131
137
  description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
132
138
  email: info@elasticsearch.com
133
139
  executables: []
@@ -164,9 +170,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
164
170
  - - '>='
165
171
  - !ruby/object:Gem::Version
166
172
  version: '0'
167
- requirements:
168
- - jar 'org.apache.kafka:kafka_2.9.2', '0.8.1.1'
169
- - jar 'org.slf4j:slf4j-log4j12', '1.7.10'
173
+ requirements: []
170
174
  rubyforge_project:
171
175
  rubygems_version: 2.1.9
172
176
  signing_key: