logstash-integration-kafka 10.4.0-java → 10.5.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6ebbcd2d18d130e9fac997330c3c4b4bd9a959a982fe83215762b03638497ba4
-  data.tar.gz: 2b54ba231d9f74344a5ec321e0dcdec256ea5664001e7c3dc0323b2150761e30
+  metadata.gz: c35575aafc1330e1fac14f137c818b3836f399ee1f4514a86a8a7387c4d8e8e9
+  data.tar.gz: 7fb89bca8ec2b25e07ab411b75f7de8fc4edc97f22fd2b0d5869452b10a529d9
 SHA512:
-  metadata.gz: a8c2aa5c2123fa001f58fc3670bd90face614fed72cf24f17ad645ba4de3bd689923d51ba5b5dd3a9201507657f6ed54326ed48495d274bb7c2284525470bdf7
-  data.tar.gz: cebe4abeda34edd6d5d1872c96f1b119abfa7abb2e40c52fb061e2c0953789441223e4b5a93a0d2fd7e3de1918c592dce9f5fee91bb6713b0e16f167033c13ce
+  metadata.gz: 7d0185e11c203175272ac52eb89a1fb3c1a09906832bc34dde3e2eca513575d9b47c79ca741be609c07f3d8f3d191f4371447d6987ad715516da786f7c1622a3
+  data.tar.gz: 93063bdf2cb603134865fc745c31c37a134d2fca09fac7bd9d68c798c792f471c1388fb29a600b45cc56d506c35a700a2c1236577d0fcbf1a65097cea91bf3f4

data/CHANGELOG.md

@@ -1,8 +1,14 @@
+## 10.5.0
+  - Changed: retry sending messages only for retriable exceptions [#27](https://github.com/logstash-plugins/logstash-integration-kafka/pull/29)
+
+## 10.4.1
+  - [DOC] Fixed formatting issues and made minor content edits [#43](https://github.com/logstash-plugins/logstash-integration-kafka/pull/43)
+
 ## 10.4.0
  - added the input `isolation_level` to allow fine control of whether to return transactional messages [#44](https://github.com/logstash-plugins/logstash-integration-kafka/pull/44)
 
 ## 10.3.0
-  - added the input and output `client_dns_lookup` parameter to allow control of how DNS requests are made
+  - added the input and output `client_dns_lookup` parameter to allow control of how DNS requests are made [#28](https://github.com/logstash-plugins/logstash-integration-kafka/pull/28)
 
 ## 10.2.0
   - Changed: config defaults to be aligned with Kafka client defaults [#30](https://github.com/logstash-plugins/logstash-integration-kafka/pull/30)

data/docs/output-kafka.asciidoc

@@ -115,10 +115,13 @@ output plugins.
 The number of acknowledgments the producer requires the leader to have received
 before considering a request complete.
 
-acks=0,   the producer will not wait for any acknowledgment from the server at all.
-acks=1,   This will mean the leader will write the record to its local log but
-          will respond without awaiting full acknowledgement from all followers.
-acks=all, This means the leader will wait for the full set of in-sync replicas to acknowledge the record.
+`acks=0`. The producer will not wait for any acknowledgment from the server.
+
+`acks=1`. The leader will write the record to its local log, but will respond
+without waiting for full acknowledgement from all followers.
+          
+`acks=all`. The leader will wait for the full set of in-sync replicas before
+acknowledging the record.
 
 [id="plugins-{type}s-{plugin}-batch_size"]
 ===== `batch_size` 
@@ -154,11 +157,12 @@ The total bytes of memory the producer can use to buffer records waiting to be s
 ===== `client_dns_lookup` 
 
   * Value type is <<string,string>>
+  * Valid options are `use_all_dns_ips`, `resolve_canonical_bootstrap_servers_only`, `default`
   * Default value is `"default"`
 
-How DNS lookups should be done. If set to `use_all_dns_ips`, when the lookup returns multiple 
-IP addresses for a hostname, they will all be attempted to connect to before failing the 
-connection. If the value is `resolve_canonical_bootstrap_servers_only` each entry will be 
+Controls how DNS lookups are done. If set to `use_all_dns_ips`, Logstash tries
+all IP addresses returned for a hostname before failing the connection.
+If set to `resolve_canonical_bootstrap_servers_only`, each entry will be 
 resolved and expanded into a list of canonical names.
 
 [id="plugins-{type}s-{plugin}-client_id"]
@@ -178,7 +182,7 @@ ip/port by allowing a logical application name to be included with the request
   * Default value is `"none"`
 
 The compression type for all data generated by the producer.
-The default is none (i.e. no compression). Valid values are none, gzip, or snappy.
+The default is none (i.e. no compression). Valid values are none, gzip, snappy, or lz4.
 
 [id="plugins-{type}s-{plugin}-jaas_path"]
 ===== `jaas_path` 
@@ -323,6 +327,15 @@ Kafka down, etc).
 
 A value less than zero is a configuration error.
 
+Starting with version 10.5.0, this plugin will only retry exceptions that are a subclass of
+https://kafka.apache.org/25/javadoc/org/apache/kafka/common/errors/RetriableException.html[RetriableException]
+and
+https://kafka.apache.org/25/javadoc/org/apache/kafka/common/errors/InterruptException.html[InterruptException].
+If producing a message throws any other exception, an error is logged and the message is dropped without retrying.
+This prevents the Logstash pipeline from hanging indefinitely.
+
+In versions prior to 10.5.0, any exception is retried indefinitely unless the `retries` option is configured.
+
 [id="plugins-{type}s-{plugin}-retry_backoff_ms"]
 ===== `retry_backoff_ms` 
 

data/lib/logstash/outputs/kafka.rb

@@ -236,7 +236,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
     remaining = @retries
 
     while batch.any?
-      if !remaining.nil?
+      unless remaining.nil?
         if remaining < 0
           # TODO(sissel): Offer to DLQ? Then again, if it's a transient fault,
           # DLQing would make things worse (you dlq data that would be successful
@@ -255,27 +255,39 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
         begin
           # send() can throw an exception even before the future is created.
           @producer.send(record)
-        rescue org.apache.kafka.common.errors.TimeoutException => e
+        rescue org.apache.kafka.common.errors.InterruptException,
+               org.apache.kafka.common.errors.RetriableException => e
+          logger.info("producer send failed, will retry sending", :exception => e.class, :message => e.message)
           failures << record
           nil
-        rescue org.apache.kafka.common.errors.InterruptException => e
-          failures << record
-          nil
-        rescue org.apache.kafka.common.errors.SerializationException => e
-          # TODO(sissel): Retrying will fail because the data itself has a problem serializing.
-          # TODO(sissel): Let's add DLQ here.
-          failures << record
+        rescue org.apache.kafka.common.KafkaException => e
+          # This error is not retriable, drop event
+          # TODO: add DLQ support
+          logger.warn("producer send failed, dropping record",:exception => e.class, :message => e.message,
+                      :record_value => record.value)
           nil
         end
-      end.compact
+      end
 
       futures.each_with_index do |future, i|
-        begin
-          result = future.get()
-        rescue => e
-          # TODO(sissel): Add metric to count failures, possibly by exception type.
-          logger.warn("producer send failed", :exception => e.class, :message => e.message)
-          failures << batch[i]
+        # We cannot skip nils using `futures.compact` because then our index `i` will not align with `batch`
+        unless future.nil?
+          begin
+            future.get
+          rescue java.util.concurrent.ExecutionException => e
+            # TODO(sissel): Add metric to count failures, possibly by exception type.
+            if e.get_cause.is_a? org.apache.kafka.common.errors.RetriableException or
+               e.get_cause.is_a? org.apache.kafka.common.errors.InterruptException
+              logger.info("producer send failed, will retry sending", :exception => e.cause.class,
+                          :message => e.cause.message)
+              failures << batch[i]
+            elsif e.get_cause.is_a? org.apache.kafka.common.KafkaException
+              # This error is not retriable, drop event
+              # TODO: add DLQ support
+              logger.warn("producer send failed, dropping record", :exception => e.cause.class,
+                          :message => e.cause.message, :record_value => batch[i].value)
+            end
+          end
         end
       end
 

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '10.4.0'
+  s.version         = '10.5.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+

data/spec/unit/outputs/kafka_spec.rb

@@ -56,14 +56,15 @@ describe "outputs/kafka" do
     end
   end
 
-  context "when KafkaProducer#send() raises an exception" do
+  context "when KafkaProducer#send() raises a retriable exception" do
     let(:failcount) { (rand * 10).to_i }
     let(:sendcount) { failcount + 1 }
 
     let(:exception_classes) { [
       org.apache.kafka.common.errors.TimeoutException,
+      org.apache.kafka.common.errors.DisconnectException,
+      org.apache.kafka.common.errors.CoordinatorNotAvailableException,
       org.apache.kafka.common.errors.InterruptException,
-      org.apache.kafka.common.errors.SerializationException
     ] }
 
     before do
@@ -88,6 +89,37 @@ describe "outputs/kafka" do
     end
   end
 
+  context "when KafkaProducer#send() raises a non-retriable exception" do
+    let(:failcount) { (rand * 10).to_i }
+
+    let(:exception_classes) { [
+        org.apache.kafka.common.errors.SerializationException,
+        org.apache.kafka.common.errors.RecordTooLargeException,
+        org.apache.kafka.common.errors.InvalidTopicException
+    ] }
+
+    before do
+      count = 0
+      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
+        .exactly(1).times
+        .and_wrap_original do |m, *args|
+        if count < failcount # fail 'failcount' times in a row.
+          count += 1
+          # Pick an exception at random
+          raise exception_classes.shuffle.first.new("injected exception for testing")
+        else
+          m.call(*args) # call original
+        end
+      end
+    end
+
+    it "should not retry" do
+      kafka = LogStash::Outputs::Kafka.new(simple_kafka_config)
+      kafka.register
+      kafka.multi_receive([event])
+    end
+  end
+
   context "when a send fails" do
     context "and the default retries behavior is used" do
       # Fail this many times and then finally succeed.
@@ -107,7 +139,7 @@ describe "outputs/kafka" do
             # inject some failures.
 
             # Return a custom Future that will raise an exception to simulate a Kafka send() problem.
-            future = java.util.concurrent.FutureTask.new { raise "Failed" }
+            future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
             future.run
             future
           else
@@ -129,7 +161,7 @@ describe "outputs/kafka" do
                                                                                        .once
                                                                                        .and_wrap_original do |m, *args|
           # Always fail.
-          future = java.util.concurrent.FutureTask.new { raise "Failed" }
+          future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
           future.run
           future
         end
@@ -143,7 +175,7 @@ describe "outputs/kafka" do
                                                                                        .once
                                                                                        .and_wrap_original do |m, *args|
           # Always fail.
-          future = java.util.concurrent.FutureTask.new { raise "Failed" }
+          future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
           future.run
           future
         end
@@ -164,7 +196,7 @@ describe "outputs/kafka" do
               .at_most(max_sends).times
               .and_wrap_original do |m, *args|
           # Always fail.
-          future = java.util.concurrent.FutureTask.new { raise "Failed" }
+          future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
           future.run
           future
         end
@@ -175,10 +207,10 @@ describe "outputs/kafka" do
 
       it 'should only sleep retries number of times' do
         expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
-                                                                                       .at_most(max_sends)
+                                                                                       .at_most(max_sends).times
                                                                                        .and_wrap_original do |m, *args|
           # Always fail.
-          future = java.util.concurrent.FutureTask.new { raise "Failed" }
+          future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
           future.run
           future
         end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-kafka
 version: !ruby/object:Gem::Version
-  version: 10.4.0
+  version: 10.5.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-07-03 00:00:00.000000000 Z
+date: 2020-07-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement