logstash-input-kafka 2.0.3 → 2.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +4 -0
  3. data/lib/logstash/inputs/kafka.rb +9 -0
  4. data/logstash-input-kafka.gemspec +2 -2
  5. data/spec/inputs/kafka_spec.rb +1 -1
  6. metadata +30 -36
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: b3867a2da2e1bcef41af77b310020da1c2a8cc78
4
- data.tar.gz: d43d020db142194bd90d112b61ae09a80a6e5f73
3
+ metadata.gz: 8cbf018ee79064fe87ed7e0a6ec383bee0d552f5
4
+ data.tar.gz: e536bdb136d069db76dd7ff890a91c3cbbb24902
5
5
  SHA512:
6
- metadata.gz: 319106e8a1dca73d8d4fca0db83b1835061e65fa14f5d0e37e092b64356b17d32cec30d01b88c60a415496f34a950e7915aa17f97bac83ecdce0793540f8190e
7
- data.tar.gz: 780f0dbed7c2662e6dbffe1320155efa75de10c5899f8eec5a33012750a6d28ecb136cf0677b8aa8afeb1d9918bfc27f97fc7e5ee441b1ba2db4e7235814404b
6
+ metadata.gz: 50e38271ea18c0ca7edb4d1d16f3be5025f4ec0d6b204bdf7663af0831a70ea50cdc8e8a5a700f721d2d4b6656ac70989b1bfb43198fdc1b14a97fba2af023ba
7
+ data.tar.gz: 7e4212059e99825fbc47ee842826727dc78a2c74459a5a89ba3a7cdb8e9d0583e5db0cf16c3e0ac1a5079df4ccd1dd32566cdc9c702a00b6d388bf36f199662c
data/CHANGELOG.md CHANGED
@@ -1,3 +1,7 @@
1
+ ## 2.0.4
2
+ - Fix safe shutdown while plugin waits on Kafka for new events
3
+ - Expose auto_commit_interval_ms to control offset commit frequency
4
+
1
5
  ## 2.0.3
2
6
  - Fix infinite loop when no new messages are found in Kafka
3
7
 
data/lib/logstash/inputs/kafka.rb CHANGED
@@ -50,6 +50,8 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
50
50
  # have an established offset or offset is invalid, start with the earliest message present in the
51
51
  # log (`smallest`) or after the last message in the log (`largest`).
52
52
  config :auto_offset_reset, :validate => %w( largest smallest ), :default => 'largest'
53
+ # The frequency in ms that the consumer offsets are committed to zookeeper.
54
+ config :auto_commit_interval_ms, :validate => :number, :default => 1000
53
55
  # Number of threads to read from the partitions. Ideally you should have as many threads as the
54
56
  # number of partitions for a perfect balance. More threads than partitions means that some
55
57
  # threads will be idle. Less threads means a single thread could be consuming from more than
@@ -92,6 +94,8 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
92
94
  # The serializer class for keys (defaults to the same default as for messages)
93
95
  config :key_decoder_class, :validate => :string, :default => 'kafka.serializer.DefaultDecoder'
94
96
 
97
+ class KafkaShutdownEvent; end
98
+ KAFKA_SHUTDOWN_EVENT = KafkaShutdownEvent.new
95
99
 
96
100
  public
97
101
  def register
@@ -101,6 +105,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
101
105
  :group_id => @group_id,
102
106
  :topic_id => @topic_id,
103
107
  :auto_offset_reset => @auto_offset_reset,
108
+ :auto_commit_interval => @auto_commit_interval_ms,
104
109
  :rebalance_max_retries => @rebalance_max_retries,
105
110
  :rebalance_backoff_ms => @rebalance_backoff_ms,
106
111
  :consumer_timeout_ms => @consumer_timeout_ms,
@@ -137,6 +142,9 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
137
142
 
138
143
  while !stop?
139
144
  event = @kafka_client_queue.pop
145
+ if event == KAFKA_SHUTDOWN_EVENT
146
+ break
147
+ end
140
148
  queue_event(event, logstash_queue)
141
149
  end
142
150
 
@@ -155,6 +163,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
155
163
 
156
164
  public
157
165
  def stop
166
+ @kafka_client_queue.push(KAFKA_SHUTDOWN_EVENT)
158
167
  @consumer_group.shutdown if @consumer_group.running?
159
168
  end
160
169
 
data/logstash-input-kafka.gemspec CHANGED
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-input-kafka'
4
- s.version = '2.0.3'
4
+ s.version = '2.0.4'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = 'This input will read events from a Kafka topic. It uses the high level consumer API provided by Kafka to read messages from the broker'
7
7
  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -25,7 +25,7 @@ Gem::Specification.new do |s|
25
25
  s.add_runtime_dependency 'logstash-codec-plain'
26
26
  s.add_runtime_dependency 'stud', '>= 0.0.22', '< 0.1.0'
27
27
 
28
- s.add_runtime_dependency 'jruby-kafka', ['>= 1.2.0', '< 2.0.0']
28
+ s.add_runtime_dependency 'jruby-kafka', '1.5.0'
29
29
 
30
30
  s.add_development_dependency 'logstash-devutils'
31
31
  end
data/spec/inputs/kafka_spec.rb CHANGED
@@ -42,7 +42,7 @@ class TestInfiniteKafkaGroup < Kafka::Group
42
42
  Thread.new do
43
43
  while true
44
44
  a_queue << blah
45
- sleep 0.2
45
+ sleep 10
46
46
  end
47
47
  end
48
48
  end
metadata CHANGED
@@ -1,17 +1,18 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-input-kafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.3
4
+ version: 2.0.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elasticsearch
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-12-14 00:00:00.000000000 Z
11
+ date: 2016-01-27 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- requirement: !ruby/object:Gem::Requirement
14
+ name: logstash-core
15
+ version_requirements: !ruby/object:Gem::Requirement
15
16
  requirements:
16
17
  - - '>='
17
18
  - !ruby/object:Gem::Version
@@ -19,10 +20,7 @@ dependencies:
19
20
  - - <
20
21
  - !ruby/object:Gem::Version
21
22
  version: 3.0.0
22
- name: logstash-core
23
- prerelease: false
24
- type: :runtime
25
- version_requirements: !ruby/object:Gem::Requirement
23
+ requirement: !ruby/object:Gem::Requirement
26
24
  requirements:
27
25
  - - '>='
28
26
  - !ruby/object:Gem::Version
@@ -30,46 +28,38 @@ dependencies:
30
28
  - - <
31
29
  - !ruby/object:Gem::Version
32
30
  version: 3.0.0
33
- - !ruby/object:Gem::Dependency
34
- requirement: !ruby/object:Gem::Requirement
35
- requirements:
36
- - - '>='
37
- - !ruby/object:Gem::Version
38
- version: '0'
39
- name: logstash-codec-json
40
31
  prerelease: false
41
32
  type: :runtime
33
+ - !ruby/object:Gem::Dependency
34
+ name: logstash-codec-json
42
35
  version_requirements: !ruby/object:Gem::Requirement
43
36
  requirements:
44
37
  - - '>='
45
38
  - !ruby/object:Gem::Version
46
39
  version: '0'
47
- - !ruby/object:Gem::Dependency
48
40
  requirement: !ruby/object:Gem::Requirement
49
41
  requirements:
50
42
  - - '>='
51
43
  - !ruby/object:Gem::Version
52
44
  version: '0'
53
- name: logstash-codec-plain
54
45
  prerelease: false
55
46
  type: :runtime
47
+ - !ruby/object:Gem::Dependency
48
+ name: logstash-codec-plain
56
49
  version_requirements: !ruby/object:Gem::Requirement
57
50
  requirements:
58
51
  - - '>='
59
52
  - !ruby/object:Gem::Version
60
53
  version: '0'
61
- - !ruby/object:Gem::Dependency
62
54
  requirement: !ruby/object:Gem::Requirement
63
55
  requirements:
64
56
  - - '>='
65
57
  - !ruby/object:Gem::Version
66
- version: 0.0.22
67
- - - <
68
- - !ruby/object:Gem::Version
69
- version: 0.1.0
70
- name: stud
58
+ version: '0'
71
59
  prerelease: false
72
60
  type: :runtime
61
+ - !ruby/object:Gem::Dependency
62
+ name: stud
73
63
  version_requirements: !ruby/object:Gem::Requirement
74
64
  requirements:
75
65
  - - '>='
@@ -78,40 +68,44 @@ dependencies:
78
68
  - - <
79
69
  - !ruby/object:Gem::Version
80
70
  version: 0.1.0
81
- - !ruby/object:Gem::Dependency
82
71
  requirement: !ruby/object:Gem::Requirement
83
72
  requirements:
84
73
  - - '>='
85
74
  - !ruby/object:Gem::Version
86
- version: 1.2.0
75
+ version: 0.0.22
87
76
  - - <
88
77
  - !ruby/object:Gem::Version
89
- version: 2.0.0
90
- name: jruby-kafka
78
+ version: 0.1.0
91
79
  prerelease: false
92
80
  type: :runtime
81
+ - !ruby/object:Gem::Dependency
82
+ name: jruby-kafka
93
83
  version_requirements: !ruby/object:Gem::Requirement
94
84
  requirements:
95
- - - '>='
85
+ - - '='
96
86
  - !ruby/object:Gem::Version
97
- version: 1.2.0
98
- - - <
87
+ version: 1.5.0
88
+ requirement: !ruby/object:Gem::Requirement
89
+ requirements:
90
+ - - '='
99
91
  - !ruby/object:Gem::Version
100
- version: 2.0.0
92
+ version: 1.5.0
93
+ prerelease: false
94
+ type: :runtime
101
95
  - !ruby/object:Gem::Dependency
102
- requirement: !ruby/object:Gem::Requirement
96
+ name: logstash-devutils
97
+ version_requirements: !ruby/object:Gem::Requirement
103
98
  requirements:
104
99
  - - '>='
105
100
  - !ruby/object:Gem::Version
106
101
  version: '0'
107
- name: logstash-devutils
108
- prerelease: false
109
- type: :development
110
- version_requirements: !ruby/object:Gem::Requirement
102
+ requirement: !ruby/object:Gem::Requirement
111
103
  requirements:
112
104
  - - '>='
113
105
  - !ruby/object:Gem::Version
114
106
  version: '0'
107
+ prerelease: false
108
+ type: :development
115
109
  description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
116
110
  email: info@elastic.co
117
111
  executables: []
@@ -150,7 +144,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
150
144
  version: '0'
151
145
  requirements: []
152
146
  rubyforge_project:
153
- rubygems_version: 2.4.5
147
+ rubygems_version: 2.4.8
154
148
  signing_key:
155
149
  specification_version: 4
156
150
  summary: This input will read events from a Kafka topic. It uses the high level consumer API provided by Kafka to read messages from the broker