logstash-integration-kafka 10.4.0-java → 10.5.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 6ebbcd2d18d130e9fac997330c3c4b4bd9a959a982fe83215762b03638497ba4
4
- data.tar.gz: 2b54ba231d9f74344a5ec321e0dcdec256ea5664001e7c3dc0323b2150761e30
3
+ metadata.gz: c35575aafc1330e1fac14f137c818b3836f399ee1f4514a86a8a7387c4d8e8e9
4
+ data.tar.gz: 7fb89bca8ec2b25e07ab411b75f7de8fc4edc97f22fd2b0d5869452b10a529d9
5
5
  SHA512:
6
- metadata.gz: a8c2aa5c2123fa001f58fc3670bd90face614fed72cf24f17ad645ba4de3bd689923d51ba5b5dd3a9201507657f6ed54326ed48495d274bb7c2284525470bdf7
7
- data.tar.gz: cebe4abeda34edd6d5d1872c96f1b119abfa7abb2e40c52fb061e2c0953789441223e4b5a93a0d2fd7e3de1918c592dce9f5fee91bb6713b0e16f167033c13ce
6
+ metadata.gz: 7d0185e11c203175272ac52eb89a1fb3c1a09906832bc34dde3e2eca513575d9b47c79ca741be609c07f3d8f3d191f4371447d6987ad715516da786f7c1622a3
7
+ data.tar.gz: 93063bdf2cb603134865fc745c31c37a134d2fca09fac7bd9d68c798c792f471c1388fb29a600b45cc56d506c35a700a2c1236577d0fcbf1a65097cea91bf3f4
@@ -1,8 +1,14 @@
1
+ ## 10.5.0
2
+ - Changed: retry sending messages only for retriable exceptions [#27](https://github.com/logstash-plugins/logstash-integration-kafka/pull/29)
3
+
4
+ ## 10.4.1
5
+ - [DOC] Fixed formatting issues and made minor content edits [#43](https://github.com/logstash-plugins/logstash-integration-kafka/pull/43)
6
+
1
7
  ## 10.4.0
2
8
  - added the input `isolation_level` to allow fine control of whether to return transactional messages [#44](https://github.com/logstash-plugins/logstash-integration-kafka/pull/44)
3
9
 
4
10
  ## 10.3.0
5
- - added the input and output `client_dns_lookup` parameter to allow control of how DNS requests are made
11
+ - added the input and output `client_dns_lookup` parameter to allow control of how DNS requests are made [#28](https://github.com/logstash-plugins/logstash-integration-kafka/pull/28)
6
12
 
7
13
  ## 10.2.0
8
14
  - Changed: config defaults to be aligned with Kafka client defaults [#30](https://github.com/logstash-plugins/logstash-integration-kafka/pull/30)
@@ -115,10 +115,13 @@ output plugins.
115
115
  The number of acknowledgments the producer requires the leader to have received
116
116
  before considering a request complete.
117
117
 
118
- acks=0, the producer will not wait for any acknowledgment from the server at all.
119
- acks=1, This will mean the leader will write the record to its local log but
120
- will respond without awaiting full acknowledgement from all followers.
121
- acks=all, This means the leader will wait for the full set of in-sync replicas to acknowledge the record.
118
+ `acks=0`. The producer will not wait for any acknowledgment from the server.
119
+
120
+ `acks=1`. The leader will write the record to its local log, but will respond
121
+ without waiting for full acknowledgement from all followers.
122
+
123
+ `acks=all`. The leader will wait for the full set of in-sync replicas before
124
+ acknowledging the record.
122
125
 
123
126
  [id="plugins-{type}s-{plugin}-batch_size"]
124
127
  ===== `batch_size`
@@ -154,11 +157,12 @@ The total bytes of memory the producer can use to buffer records waiting to be s
154
157
  ===== `client_dns_lookup`
155
158
 
156
159
  * Value type is <<string,string>>
160
+ * Valid options are `use_all_dns_ips`, `resolve_canonical_bootstrap_servers_only`, `default`
157
161
  * Default value is `"default"`
158
162
 
159
- How DNS lookups should be done. If set to `use_all_dns_ips`, when the lookup returns multiple
160
- IP addresses for a hostname, they will all be attempted to connect to before failing the
161
- connection. If the value is `resolve_canonical_bootstrap_servers_only` each entry will be
163
+ Controls how DNS lookups are done. If set to `use_all_dns_ips`, Logstash tries
164
+ all IP addresses returned for a hostname before failing the connection.
165
+ If set to `resolve_canonical_bootstrap_servers_only`, each entry will be
162
166
  resolved and expanded into a list of canonical names.
163
167
 
164
168
  [id="plugins-{type}s-{plugin}-client_id"]
@@ -178,7 +182,7 @@ ip/port by allowing a logical application name to be included with the request
178
182
  * Default value is `"none"`
179
183
 
180
184
  The compression type for all data generated by the producer.
181
- The default is none (i.e. no compression). Valid values are none, gzip, or snappy.
185
+ The default is none (i.e. no compression). Valid values are none, gzip, snappy, or lz4.
182
186
 
183
187
  [id="plugins-{type}s-{plugin}-jaas_path"]
184
188
  ===== `jaas_path`
@@ -323,6 +327,15 @@ Kafka down, etc).
323
327
 
324
328
  A value less than zero is a configuration error.
325
329
 
330
+ Starting with version 10.5.0, this plugin will only retry exceptions that are a subclass of
331
+ https://kafka.apache.org/25/javadoc/org/apache/kafka/common/errors/RetriableException.html[RetriableException]
332
+ and
333
+ https://kafka.apache.org/25/javadoc/org/apache/kafka/common/errors/InterruptException.html[InterruptException].
334
+ If producing a message throws any other exception, an error is logged and the message is dropped without retrying.
335
+ This prevents the Logstash pipeline from hanging indefinitely.
336
+
337
+ In versions prior to 10.5.0, any exception is retried indefinitely unless the `retries` option is configured.
338
+
326
339
  [id="plugins-{type}s-{plugin}-retry_backoff_ms"]
327
340
  ===== `retry_backoff_ms`
328
341
 
@@ -236,7 +236,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
236
236
  remaining = @retries
237
237
 
238
238
  while batch.any?
239
- if !remaining.nil?
239
+ unless remaining.nil?
240
240
  if remaining < 0
241
241
  # TODO(sissel): Offer to DLQ? Then again, if it's a transient fault,
242
242
  # DLQing would make things worse (you dlq data that would be successful
@@ -255,27 +255,39 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
255
255
  begin
256
256
  # send() can throw an exception even before the future is created.
257
257
  @producer.send(record)
258
- rescue org.apache.kafka.common.errors.TimeoutException => e
258
+ rescue org.apache.kafka.common.errors.InterruptException,
259
+ org.apache.kafka.common.errors.RetriableException => e
260
+ logger.info("producer send failed, will retry sending", :exception => e.class, :message => e.message)
259
261
  failures << record
260
262
  nil
261
- rescue org.apache.kafka.common.errors.InterruptException => e
262
- failures << record
263
- nil
264
- rescue org.apache.kafka.common.errors.SerializationException => e
265
- # TODO(sissel): Retrying will fail because the data itself has a problem serializing.
266
- # TODO(sissel): Let's add DLQ here.
267
- failures << record
263
+ rescue org.apache.kafka.common.KafkaException => e
264
+ # This error is not retriable, drop event
265
+ # TODO: add DLQ support
266
+ logger.warn("producer send failed, dropping record",:exception => e.class, :message => e.message,
267
+ :record_value => record.value)
268
268
  nil
269
269
  end
270
- end.compact
270
+ end
271
271
 
272
272
  futures.each_with_index do |future, i|
273
- begin
274
- result = future.get()
275
- rescue => e
276
- # TODO(sissel): Add metric to count failures, possibly by exception type.
277
- logger.warn("producer send failed", :exception => e.class, :message => e.message)
278
- failures << batch[i]
273
+ # We cannot skip nils using `futures.compact` because then our index `i` will not align with `batch`
274
+ unless future.nil?
275
+ begin
276
+ future.get
277
+ rescue java.util.concurrent.ExecutionException => e
278
+ # TODO(sissel): Add metric to count failures, possibly by exception type.
279
+ if e.get_cause.is_a? org.apache.kafka.common.errors.RetriableException or
280
+ e.get_cause.is_a? org.apache.kafka.common.errors.InterruptException
281
+ logger.info("producer send failed, will retry sending", :exception => e.cause.class,
282
+ :message => e.cause.message)
283
+ failures << batch[i]
284
+ elsif e.get_cause.is_a? org.apache.kafka.common.KafkaException
285
+ # This error is not retriable, drop event
286
+ # TODO: add DLQ support
287
+ logger.warn("producer send failed, dropping record", :exception => e.cause.class,
288
+ :message => e.cause.message, :record_value => batch[i].value)
289
+ end
290
+ end
279
291
  end
280
292
  end
281
293
 
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-integration-kafka'
3
- s.version = '10.4.0'
3
+ s.version = '10.5.0'
4
4
  s.licenses = ['Apache-2.0']
5
5
  s.summary = "Integration with Kafka - input and output plugins"
6
6
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+
@@ -56,14 +56,15 @@ describe "outputs/kafka" do
56
56
  end
57
57
  end
58
58
 
59
- context "when KafkaProducer#send() raises an exception" do
59
+ context "when KafkaProducer#send() raises a retriable exception" do
60
60
  let(:failcount) { (rand * 10).to_i }
61
61
  let(:sendcount) { failcount + 1 }
62
62
 
63
63
  let(:exception_classes) { [
64
64
  org.apache.kafka.common.errors.TimeoutException,
65
+ org.apache.kafka.common.errors.DisconnectException,
66
+ org.apache.kafka.common.errors.CoordinatorNotAvailableException,
65
67
  org.apache.kafka.common.errors.InterruptException,
66
- org.apache.kafka.common.errors.SerializationException
67
68
  ] }
68
69
 
69
70
  before do
@@ -88,6 +89,37 @@ describe "outputs/kafka" do
88
89
  end
89
90
  end
90
91
 
92
+ context "when KafkaProducer#send() raises a non-retriable exception" do
93
+ let(:failcount) { (rand * 10).to_i }
94
+
95
+ let(:exception_classes) { [
96
+ org.apache.kafka.common.errors.SerializationException,
97
+ org.apache.kafka.common.errors.RecordTooLargeException,
98
+ org.apache.kafka.common.errors.InvalidTopicException
99
+ ] }
100
+
101
+ before do
102
+ count = 0
103
+ expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
104
+ .exactly(1).times
105
+ .and_wrap_original do |m, *args|
106
+ if count < failcount # fail 'failcount' times in a row.
107
+ count += 1
108
+ # Pick an exception at random
109
+ raise exception_classes.shuffle.first.new("injected exception for testing")
110
+ else
111
+ m.call(*args) # call original
112
+ end
113
+ end
114
+ end
115
+
116
+ it "should not retry" do
117
+ kafka = LogStash::Outputs::Kafka.new(simple_kafka_config)
118
+ kafka.register
119
+ kafka.multi_receive([event])
120
+ end
121
+ end
122
+
91
123
  context "when a send fails" do
92
124
  context "and the default retries behavior is used" do
93
125
  # Fail this many times and then finally succeed.
@@ -107,7 +139,7 @@ describe "outputs/kafka" do
107
139
  # inject some failures.
108
140
 
109
141
  # Return a custom Future that will raise an exception to simulate a Kafka send() problem.
110
- future = java.util.concurrent.FutureTask.new { raise "Failed" }
142
+ future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
111
143
  future.run
112
144
  future
113
145
  else
@@ -129,7 +161,7 @@ describe "outputs/kafka" do
129
161
  .once
130
162
  .and_wrap_original do |m, *args|
131
163
  # Always fail.
132
- future = java.util.concurrent.FutureTask.new { raise "Failed" }
164
+ future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
133
165
  future.run
134
166
  future
135
167
  end
@@ -143,7 +175,7 @@ describe "outputs/kafka" do
143
175
  .once
144
176
  .and_wrap_original do |m, *args|
145
177
  # Always fail.
146
- future = java.util.concurrent.FutureTask.new { raise "Failed" }
178
+ future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
147
179
  future.run
148
180
  future
149
181
  end
@@ -164,7 +196,7 @@ describe "outputs/kafka" do
164
196
  .at_most(max_sends).times
165
197
  .and_wrap_original do |m, *args|
166
198
  # Always fail.
167
- future = java.util.concurrent.FutureTask.new { raise "Failed" }
199
+ future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
168
200
  future.run
169
201
  future
170
202
  end
@@ -175,10 +207,10 @@ describe "outputs/kafka" do
175
207
 
176
208
  it 'should only sleep retries number of times' do
177
209
  expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
178
- .at_most(max_sends)
210
+ .at_most(max_sends).times
179
211
  .and_wrap_original do |m, *args|
180
212
  # Always fail.
181
- future = java.util.concurrent.FutureTask.new { raise "Failed" }
213
+ future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
182
214
  future.run
183
215
  future
184
216
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-integration-kafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 10.4.0
4
+ version: 10.5.0
5
5
  platform: java
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-07-03 00:00:00.000000000 Z
11
+ date: 2020-07-20 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement