logstash-integration-kafka 10.2.0-java → 10.5.2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 88d87b3c5b9e443c0bc4ba8064b6316243f24bb0c0d6ce150ca0a19dd25ecb2a
-  data.tar.gz: 5fc09468e8cdc7094bc6099c24d70b1c19bc339c3ec3dfa3562e0c389fa85753
+  metadata.gz: 877f38709d9280199d61c2c7755461223a9695bd966d8d7ad77676b40c071102
+  data.tar.gz: 34d9f91fc7ae51cc4419ef19ada731b8a122006338371daf0d5c6b23c692a46c
 SHA512:
-  metadata.gz: 4e0fc4586ca91967e9f8afa4a2eef95b7a5f8cd13f3b6689b4ace12b64d3ea628bc5112735d54edcb20c0fe1c6debac1e4d32983e1555eb9a84d380ab5d68a4e
-  data.tar.gz: 10ae655222832b9b5f6b9d8aa155d0907da91e38f04c7ad74c949d7bf4c8ff5d222bde6b2e1779eca5bd4bebb99aaf54701c8e2fddd4cb5a3cff47fabe9831fe
+  metadata.gz: e1587ee36b4e8038c17c8ed9182d7e25e040447df017917bc4b8dc69f1ffe0c3ab9c624bccabe6ba4439d9ac7ffa38dfab417ac93ee46e0aef6308a303ba5dee
+  data.tar.gz: 45c48cf5e734e86e9454b4084a83289d89ae51613d100c8b02587d7be4ce9914cb7956b0a2fa51bfd1dea2d9929081becf9ebc1c5f90cf4e03d6076bf3ed8f63

data/CHANGELOG.md

@@ -1,3 +1,23 @@
+## 10.5.2
+  - Docs: explain group_id in case of multiple inputs [#59](https://github.com/logstash-plugins/logstash-integration-kafka/pull/59)
+
+## 10.5.1
+  - [DOC]Replaced plugin_header file with plugin_header-integration file. [#46](https://github.com/logstash-plugins/logstash-integration-kafka/pull/46)
+  - [DOC]Update kafka client version across kafka integration docs [#47](https://github.com/logstash-plugins/logstash-integration-kafka/pull/47)
+  - [DOC]Replace hard-coded kafka client and doc path version numbers with attributes to simplify doc maintenance [#48](https://github.com/logstash-plugins/logstash-integration-kafka/pull/48)  
+
+## 10.5.0
+  - Changed: retry sending messages only for retriable exceptions [#27](https://github.com/logstash-plugins/logstash-integration-kafka/pull/29)
+
+## 10.4.1
+  - [DOC] Fixed formatting issues and made minor content edits [#43](https://github.com/logstash-plugins/logstash-integration-kafka/pull/43)
+
+## 10.4.0
+ - added the input `isolation_level` to allow fine control of whether to return transactional messages [#44](https://github.com/logstash-plugins/logstash-integration-kafka/pull/44)
+
+## 10.3.0
+  - added the input and output `client_dns_lookup` parameter to allow control of how DNS requests are made [#28](https://github.com/logstash-plugins/logstash-integration-kafka/pull/28)
+
 ## 10.2.0
   - Changed: config defaults to be aligned with Kafka client defaults [#30](https://github.com/logstash-plugins/logstash-integration-kafka/pull/30)
 

data/CONTRIBUTORS

@@ -12,6 +12,7 @@ Contributors:
 * Kurt Hurtado (kurtado)
 * Ry Biesemeyer (yaauie)
 * Rob Cowart (robcowart)
+* Tim te Beek (timtebeek)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/docs/index.asciidoc

@@ -1,6 +1,7 @@
 :plugin: kafka
 :type: integration
 :no_codec:
+:kafka_client: 2.4
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!
@@ -21,11 +22,15 @@ include::{include_path}/plugin_header.asciidoc[]
 
 ==== Description
 
-The Kafka Integration Plugin provides integrated plugins for working with the https://kafka.apache.org/[Kafka] distributed streaming platform.
+The Kafka Integration Plugin provides integrated plugins for working with the
+https://kafka.apache.org/[Kafka] distributed streaming platform.
 
  - {logstash-ref}/plugins-inputs-kafka.html[Kafka Input Plugin]
  - {logstash-ref}/plugins-outputs-kafka.html[Kafka Output Plugin]
  
-This plugin uses Kafka Client 2.4. For broker compatibility, see the official https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka compatibility reference]. If the linked compatibility wiki is not up-to-date, please contact Kafka support/community to confirm compatibility.
+This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the official
+https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka
+compatibility reference]. If the linked compatibility wiki is not up-to-date,
+please contact Kafka support/community to confirm compatibility.
 
 :no_codec!:

data/docs/input-kafka.asciidoc

@@ -1,6 +1,9 @@
+:integration: kafka
 :plugin: kafka
 :type: input
 :default_codec: plain
+:kafka_client: 2.4
+:kafka_client_doc: 24
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!
@@ -17,15 +20,20 @@ END - GENERATED VARIABLES, DO NOT EDIT!
 
 === Kafka input plugin
 
-include::{include_path}/plugin_header.asciidoc[]
+include::{include_path}/plugin_header-integration.asciidoc[]
 
 ==== Description
 
 This input will read events from a Kafka topic.
 
-This plugin uses Kafka Client 2.1.0. For broker compatibility, see the official https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka compatibility reference]. If the linked compatibility wiki is not up-to-date, please contact Kafka support/community to confirm compatibility.
+This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the
+official
+https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka
+compatibility reference]. If the linked compatibility wiki is not up-to-date,
+please contact Kafka support/community to confirm compatibility.
 
-If you require features not yet available in this plugin (including client version upgrades), please file an issue with details about what you need.
+If you require features not yet available in this plugin (including client
+version upgrades), please file an issue with details about what you need.
 
 This input supports connecting to Kafka over:
 
@@ -46,9 +54,9 @@ the same `group_id`.
 Ideally you should have as many threads as the number of partitions for a perfect balance --
 more threads than partitions means that some threads will be idle
 
-For more information see https://kafka.apache.org/24/documentation.html#theconsumer
+For more information see https://kafka.apache.org/{kafka_client_doc}/documentation.html#theconsumer
 
-Kafka consumer configuration: https://kafka.apache.org/24/documentation.html#consumerconfigs
+Kafka consumer configuration: https://kafka.apache.org/{kafka_client_doc}/documentation.html#consumerconfigs
 
 ==== Metadata fields
 
@@ -59,7 +67,11 @@ The following metadata from Kafka broker are added under the `[@metadata]` field
 * `[@metadata][kafka][partition]`: Partition info for this message.
 * `[@metadata][kafka][offset]`: Original record offset for this message.
 * `[@metadata][kafka][key]`: Record key, if any.
-* `[@metadata][kafka][timestamp]`: Timestamp in the Record. Depending on your broker configuration, this can be either when the record was created (default) or when it was received by the broker. See more about property log.message.timestamp.type at https://kafka.apache.org/10/documentation.html#brokerconfigs
+* `[@metadata][kafka][timestamp]`: Timestamp in the Record. 
+Depending on your broker configuration, this can be
+either when the record was created (default) or when it was received by the
+broker. See more about property log.message.timestamp.type at
+https://kafka.apache.org/{kafka_client_doc}/documentation.html#brokerconfigs
 
 Metadata is only added to the event if the `decorate_events` option is set to true (it defaults to false).
 
@@ -73,7 +85,7 @@ This plugin supports these configuration options plus the <<plugins-{type}s-{plu
 
 NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting,
 and might change if Kafka's consumer defaults change.
-See the https://kafka.apache.org/24/documentation for more details.
+See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details.
 
 [cols="<,<,<",options="header",]
 |=======================================================================
@@ -82,6 +94,7 @@ See the https://kafka.apache.org/24/documentation for more details.
 | <<plugins-{type}s-{plugin}-auto_offset_reset>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-bootstrap_servers>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-check_crcs>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-client_dns_lookup>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-client_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-client_rack>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-connections_max_idle_ms>> |<<number,number>>|No
@@ -94,6 +107,7 @@ See the https://kafka.apache.org/24/documentation for more details.
 | <<plugins-{type}s-{plugin}-fetch_min_bytes>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-group_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-heartbeat_interval_ms>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-isolation_level>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-key_deserializer_class>> |<<string,string>>|No
@@ -174,6 +188,17 @@ Automatically check the CRC32 of the records consumed.
 This ensures no on-the-wire or on-disk corruption to the messages occurred.
 This check adds some overhead, so it may be disabled in cases seeking extreme performance.
 
+[id="plugins-{type}s-{plugin}-client_dns_lookup"]
+===== `client_dns_lookup` 
+
+  * Value type is <<string,string>>
+  * Default value is `"default"`
+
+How DNS lookups should be done. If set to `use_all_dns_ips`, when the lookup returns multiple 
+IP addresses for a hostname, they will all be attempted to connect to before failing the 
+connection. If the value is `resolve_canonical_bootstrap_servers_only` each entry will be 
+resolved and expanded into a list of canonical names.
+
 [id="plugins-{type}s-{plugin}-client_id"]
 ===== `client_id` 
 
@@ -289,7 +314,11 @@ before answering the request.
 
 The identifier of the group this consumer belongs to. Consumer group is a single logical subscriber
 that happens to be made up of multiple processors. Messages in a topic will be distributed to all
-Logstash instances with the same `group_id`
+Logstash instances with the same `group_id`.
+
+NOTE: In cases when multiple inputs are being used in a single pipeline, reading from different topics,
+it's essential to set a different `group_id => ...` for each input. Setting a unique `client_id => ...`
+is also recommended.
 
 [id="plugins-{type}s-{plugin}-heartbeat_interval_ms"]
 ===== `heartbeat_interval_ms` 
@@ -303,6 +332,17 @@ consumers join or leave the group. The value must be set lower than
 `session.timeout.ms`, but typically should be set no higher than 1/3 of that value.
 It can be adjusted even lower to control the expected time for normal rebalances.
 
+[id="plugins-{type}s-{plugin}-isolation_level"]
+===== `isolation_level` 
+
+  * Value type is <<string,string>>
+  * Default value is `"read_uncommitted"`
+
+Controls how to read messages written transactionally. If set to `read_committed`, polling messages will only return
+transactional messages which have been committed. If set to `read_uncommitted` (the default), polling messages will
+return all messages, even transactional messages which have been aborted. Non-transactional messages will be returned
+unconditionally in either mode.
+
 [id="plugins-{type}s-{plugin}-jaas_path"]
 ===== `jaas_path` 
 
@@ -397,7 +437,7 @@ partition ownership amongst consumer instances, supported options are:
 * `sticky`
 * `cooperative_sticky`
 
-These map to Kafka's corresponding https://kafka.apache.org/24/javadoc/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.html[`ConsumerPartitionAssignor`]
+These map to Kafka's corresponding https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.html[`ConsumerPartitionAssignor`]
 implementations.
 
 [id="plugins-{type}s-{plugin}-poll_timeout_ms"]

data/docs/output-kafka.asciidoc

@@ -1,6 +1,9 @@
+:integration: kafka
 :plugin: kafka
 :type: output
 :default_codec: plain
+:kafka_client: 2.4
+:kafka_client_doc: 24
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!
@@ -17,15 +20,20 @@ END - GENERATED VARIABLES, DO NOT EDIT!
 
 === Kafka output plugin
 
-include::{include_path}/plugin_header.asciidoc[]
+include::{include_path}/plugin_header-integration.asciidoc[]
 
 ==== Description
 
 Write events to a Kafka topic. 
 
-This plugin uses Kafka Client 2.1.0. For broker compatibility, see the official https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka compatibility reference]. If the linked compatibility wiki is not up-to-date, please contact Kafka support/community to confirm compatibility.
+This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the
+official
+https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka
+compatibility reference]. If the linked compatibility wiki is not up-to-date,
+please contact Kafka support/community to confirm compatibility.
 
-If you require features not yet available in this plugin (including client version upgrades), please file an issue with details about what you need.
+If you require features not yet available in this plugin (including client
+version upgrades), please file an issue with details about what you need.
 
 This output supports connecting to Kafka over:
 
@@ -36,9 +44,12 @@ By default security is disabled but can be turned on as needed.
 
 The only required configuration is the topic_id. 
 
-The default codec is plain. Logstash will encode your events with not only the message field but also with a timestamp and hostname.
+The default codec is plain. Logstash will encode your events with not only the
+message field but also with a timestamp and hostname.
+
+If you want the full content of your events to be sent as json, you should set
+the codec in the output configuration like this:
 
-If you want the full content of your events to be sent as json, you should set the codec in the output configuration like this:
 [source,ruby]
     output {
       kafka {
@@ -47,9 +58,11 @@ If you want the full content of your events to be sent as json, you should set t
       }
     }
     
-For more information see https://kafka.apache.org/24/documentation.html#theproducer
+For more information see
+https://kafka.apache.org/{kafka_client_doc}/documentation.html#theproducer
 
-Kafka producer configuration: https://kafka.apache.org/24/documentation.html#producerconfigs
+Kafka producer configuration:
+https://kafka.apache.org/{kafka_client_doc}/documentation.html#producerconfigs
 
 [id="plugins-{type}s-{plugin}-options"]
 ==== Kafka Output Configuration Options
@@ -58,7 +71,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 
 NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting,
 and might change if Kafka's producer defaults change.
-See the https://kafka.apache.org/24/documentation for more details.
+See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details.
 
 [cols="<,<,<",options="header",]
 |=======================================================================
@@ -67,6 +80,7 @@ See the https://kafka.apache.org/24/documentation for more details.
 | <<plugins-{type}s-{plugin}-batch_size>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-bootstrap_servers>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-buffer_memory>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-client_dns_lookup>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-client_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-compression_type>> |<<string,string>>, one of `["none", "gzip", "snappy", "lz4"]`|No
 | <<plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No
@@ -114,10 +128,13 @@ output plugins.
 The number of acknowledgments the producer requires the leader to have received
 before considering a request complete.
 
-acks=0,   the producer will not wait for any acknowledgment from the server at all.
-acks=1,   This will mean the leader will write the record to its local log but
-          will respond without awaiting full acknowledgement from all followers.
-acks=all, This means the leader will wait for the full set of in-sync replicas to acknowledge the record.
+`acks=0`. The producer will not wait for any acknowledgment from the server.
+
+`acks=1`. The leader will write the record to its local log, but will respond
+without waiting for full acknowledgement from all followers.
+          
+`acks=all`. The leader will wait for the full set of in-sync replicas before
+acknowledging the record.
 
 [id="plugins-{type}s-{plugin}-batch_size"]
 ===== `batch_size` 
@@ -149,6 +166,18 @@ subset of brokers.
 
 The total bytes of memory the producer can use to buffer records waiting to be sent to the server.
 
+[id="plugins-{type}s-{plugin}-client_dns_lookup"]
+===== `client_dns_lookup` 
+
+  * Value type is <<string,string>>
+  * Valid options are `use_all_dns_ips`, `resolve_canonical_bootstrap_servers_only`, `default`
+  * Default value is `"default"`
+
+Controls how DNS lookups are done. If set to `use_all_dns_ips`, Logstash tries
+all IP addresses returned for a hostname before failing the connection.
+If set to `resolve_canonical_bootstrap_servers_only`, each entry will be 
+resolved and expanded into a list of canonical names.
+
 [id="plugins-{type}s-{plugin}-client_id"]
 ===== `client_id` 
 
@@ -166,7 +195,7 @@ ip/port by allowing a logical application name to be included with the request
   * Default value is `"none"`
 
 The compression type for all data generated by the producer.
-The default is none (i.e. no compression). Valid values are none, gzip, or snappy.
+The default is none (i.e. no compression). Valid values are none, gzip, snappy, or lz4.
 
 [id="plugins-{type}s-{plugin}-jaas_path"]
 ===== `jaas_path` 
@@ -311,6 +340,15 @@ Kafka down, etc).
 
 A value less than zero is a configuration error.
 
+Starting with version 10.5.0, this plugin will only retry exceptions that are a subclass of
+https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/RetriableException.html[RetriableException]
+and
+https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/InterruptException.html[InterruptException].
+If producing a message throws any other exception, an error is logged and the message is dropped without retrying.
+This prevents the Logstash pipeline from hanging indefinitely.
+
+In versions prior to 10.5.0, any exception is retried indefinitely unless the `retries` option is configured.
+
 [id="plugins-{type}s-{plugin}-retry_backoff_ms"]
 ===== `retry_backoff_ms` 
 

data/lib/logstash/inputs/kafka.rb

@@ -71,6 +71,11 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   # corruption to the messages occurred. This check adds some overhead, so it may be
   # disabled in cases seeking extreme performance.
   config :check_crcs, :validate => :boolean, :default => true
+  # How DNS lookups should be done. If set to `use_all_dns_ips`, when the lookup returns multiple 
+  # IP addresses for a hostname, they will all be attempted to connect to before failing the 
+  # connection. If the value is `resolve_canonical_bootstrap_servers_only` each entry will be 
+  # resolved and expanded into a list of canonical names.
+  config :client_dns_lookup, :validate => ["default", "use_all_dns_ips", "resolve_canonical_bootstrap_servers_only"], :default => "default"
   # The id string to pass to the server when making requests. The purpose of this
   # is to be able to track the source of requests beyond just ip/port by allowing
   # a logical application name to be included.
@@ -109,6 +114,11 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   # `session.timeout.ms`, but typically should be set no higher than 1/3 of that value.
   # It can be adjusted even lower to control the expected time for normal rebalances.
   config :heartbeat_interval_ms, :validate => :number, :default => 3000 # Kafka default
+  # Controls how to read messages written transactionally. If set to read_committed, consumer.poll()
+  # will only return transactional messages which have been committed. If set to read_uncommitted'
+  # (the default), consumer.poll() will return all messages, even transactional messages which have
+  # been aborted. Non-transactional messages will be returned unconditionally in either mode.
+  config :isolation_level, :validate => ["read_uncommitted", "read_committed"], :default => "read_uncommitted" # Kafka default
   # Java Class used to deserialize the record's key
   config :key_deserializer_class, :validate => :string, :default => "org.apache.kafka.common.serialization.StringDeserializer"
   # The maximum delay between invocations of poll() when using consumer group management. This places 
@@ -296,6 +306,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
       props.put(kafka::AUTO_OFFSET_RESET_CONFIG, auto_offset_reset) unless auto_offset_reset.nil?
       props.put(kafka::BOOTSTRAP_SERVERS_CONFIG, bootstrap_servers)
       props.put(kafka::CHECK_CRCS_CONFIG, check_crcs.to_s) unless check_crcs.nil?
+      props.put(kafka::CLIENT_DNS_LOOKUP_CONFIG, client_dns_lookup)
       props.put(kafka::CLIENT_ID_CONFIG, client_id)
       props.put(kafka::CONNECTIONS_MAX_IDLE_MS_CONFIG, connections_max_idle_ms.to_s) unless connections_max_idle_ms.nil?
       props.put(kafka::ENABLE_AUTO_COMMIT_CONFIG, enable_auto_commit.to_s)
@@ -305,6 +316,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
       props.put(kafka::FETCH_MIN_BYTES_CONFIG, fetch_min_bytes.to_s) unless fetch_min_bytes.nil?
       props.put(kafka::GROUP_ID_CONFIG, group_id)
       props.put(kafka::HEARTBEAT_INTERVAL_MS_CONFIG, heartbeat_interval_ms.to_s) unless heartbeat_interval_ms.nil?
+      props.put(kafka::ISOLATION_LEVEL_CONFIG, isolation_level)
       props.put(kafka::KEY_DESERIALIZER_CLASS_CONFIG, key_deserializer_class)
       props.put(kafka::MAX_PARTITION_FETCH_BYTES_CONFIG, max_partition_fetch_bytes.to_s) unless max_partition_fetch_bytes.nil?
       props.put(kafka::MAX_POLL_RECORDS_CONFIG, max_poll_records.to_s) unless max_poll_records.nil?

data/lib/logstash/outputs/kafka.rb

@@ -79,6 +79,11 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   # The compression type for all data generated by the producer.
   # The default is none (i.e. no compression). Valid values are none, gzip, or snappy.
   config :compression_type, :validate => ["none", "gzip", "snappy", "lz4"], :default => "none"
+  # How DNS lookups should be done. If set to `use_all_dns_ips`, when the lookup returns multiple 
+  # IP addresses for a hostname, they will all be attempted to connect to before failing the 
+  # connection. If the value is `resolve_canonical_bootstrap_servers_only` each entry will be 
+  # resolved and expanded into a list of canonical names.
+  config :client_dns_lookup, :validate => ["default", "use_all_dns_ips", "resolve_canonical_bootstrap_servers_only"], :default => "default"
   # The id string to pass to the server when making requests.
   # The purpose of this is to be able to track the source of requests beyond just
   # ip/port by allowing a logical application name to be included with the request
@@ -231,7 +236,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
     remaining = @retries
 
     while batch.any?
-      if !remaining.nil?
+      unless remaining.nil?
         if remaining < 0
           # TODO(sissel): Offer to DLQ? Then again, if it's a transient fault,
           # DLQing would make things worse (you dlq data that would be successful
@@ -250,27 +255,39 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
         begin
           # send() can throw an exception even before the future is created.
           @producer.send(record)
-        rescue org.apache.kafka.common.errors.TimeoutException => e
+        rescue org.apache.kafka.common.errors.InterruptException,
+               org.apache.kafka.common.errors.RetriableException => e
+          logger.info("producer send failed, will retry sending", :exception => e.class, :message => e.message)
           failures << record
           nil
-        rescue org.apache.kafka.common.errors.InterruptException => e
-          failures << record
-          nil
-        rescue org.apache.kafka.common.errors.SerializationException => e
-          # TODO(sissel): Retrying will fail because the data itself has a problem serializing.
-          # TODO(sissel): Let's add DLQ here.
-          failures << record
+        rescue org.apache.kafka.common.KafkaException => e
+          # This error is not retriable, drop event
+          # TODO: add DLQ support
+          logger.warn("producer send failed, dropping record",:exception => e.class, :message => e.message,
+                      :record_value => record.value)
           nil
         end
-      end.compact
+      end
 
       futures.each_with_index do |future, i|
-        begin
-          result = future.get()
-        rescue => e
-          # TODO(sissel): Add metric to count failures, possibly by exception type.
-          logger.warn("producer send failed", :exception => e.class, :message => e.message)
-          failures << batch[i]
+        # We cannot skip nils using `futures.compact` because then our index `i` will not align with `batch`
+        unless future.nil?
+          begin
+            future.get
+          rescue java.util.concurrent.ExecutionException => e
+            # TODO(sissel): Add metric to count failures, possibly by exception type.
+            if e.get_cause.is_a? org.apache.kafka.common.errors.RetriableException or
+               e.get_cause.is_a? org.apache.kafka.common.errors.InterruptException
+              logger.info("producer send failed, will retry sending", :exception => e.cause.class,
+                          :message => e.cause.message)
+              failures << batch[i]
+            elsif e.get_cause.is_a? org.apache.kafka.common.KafkaException
+              # This error is not retriable, drop event
+              # TODO: add DLQ support
+              logger.warn("producer send failed, dropping record", :exception => e.cause.class,
+                          :message => e.cause.message, :record_value => batch[i].value)
+            end
+          end
         end
       end
 
@@ -318,6 +335,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
       props.put(kafka::BOOTSTRAP_SERVERS_CONFIG, bootstrap_servers)
       props.put(kafka::BUFFER_MEMORY_CONFIG, buffer_memory.to_s)
       props.put(kafka::COMPRESSION_TYPE_CONFIG, compression_type)
+      props.put(kafka::CLIENT_DNS_LOOKUP_CONFIG, client_dns_lookup)
       props.put(kafka::CLIENT_ID_CONFIG, client_id) unless client_id.nil?
       props.put(kafka::KEY_SERIALIZER_CLASS_CONFIG, key_serializer)
       props.put(kafka::LINGER_MS_CONFIG, linger_ms.to_s)

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '10.2.0'
+  s.version         = '10.5.2'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+

data/spec/unit/outputs/kafka_spec.rb

@@ -56,14 +56,15 @@ describe "outputs/kafka" do
     end
   end
 
-  context "when KafkaProducer#send() raises an exception" do
+  context "when KafkaProducer#send() raises a retriable exception" do
     let(:failcount) { (rand * 10).to_i }
     let(:sendcount) { failcount + 1 }
 
     let(:exception_classes) { [
       org.apache.kafka.common.errors.TimeoutException,
+      org.apache.kafka.common.errors.DisconnectException,
+      org.apache.kafka.common.errors.CoordinatorNotAvailableException,
       org.apache.kafka.common.errors.InterruptException,
-      org.apache.kafka.common.errors.SerializationException
     ] }
 
     before do
@@ -88,6 +89,37 @@ describe "outputs/kafka" do
     end
   end
 
+  context "when KafkaProducer#send() raises a non-retriable exception" do
+    let(:failcount) { (rand * 10).to_i }
+
+    let(:exception_classes) { [
+        org.apache.kafka.common.errors.SerializationException,
+        org.apache.kafka.common.errors.RecordTooLargeException,
+        org.apache.kafka.common.errors.InvalidTopicException
+    ] }
+
+    before do
+      count = 0
+      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
+        .exactly(1).times
+        .and_wrap_original do |m, *args|
+        if count < failcount # fail 'failcount' times in a row.
+          count += 1
+          # Pick an exception at random
+          raise exception_classes.shuffle.first.new("injected exception for testing")
+        else
+          m.call(*args) # call original
+        end
+      end
+    end
+
+    it "should not retry" do
+      kafka = LogStash::Outputs::Kafka.new(simple_kafka_config)
+      kafka.register
+      kafka.multi_receive([event])
+    end
+  end
+
   context "when a send fails" do
     context "and the default retries behavior is used" do
       # Fail this many times and then finally succeed.
@@ -107,7 +139,7 @@ describe "outputs/kafka" do
             # inject some failures.
 
             # Return a custom Future that will raise an exception to simulate a Kafka send() problem.
-            future = java.util.concurrent.FutureTask.new { raise "Failed" }
+            future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
             future.run
             future
           else
@@ -129,7 +161,7 @@ describe "outputs/kafka" do
                                                                                        .once
                                                                                        .and_wrap_original do |m, *args|
           # Always fail.
-          future = java.util.concurrent.FutureTask.new { raise "Failed" }
+          future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
           future.run
           future
         end
@@ -143,7 +175,7 @@ describe "outputs/kafka" do
                                                                                        .once
                                                                                        .and_wrap_original do |m, *args|
           # Always fail.
-          future = java.util.concurrent.FutureTask.new { raise "Failed" }
+          future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
           future.run
           future
         end
@@ -164,7 +196,7 @@ describe "outputs/kafka" do
               .at_most(max_sends).times
               .and_wrap_original do |m, *args|
           # Always fail.
-          future = java.util.concurrent.FutureTask.new { raise "Failed" }
+          future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
           future.run
           future
         end
@@ -175,10 +207,10 @@ describe "outputs/kafka" do
 
       it 'should only sleep retries number of times' do
         expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
-                                                                                       .at_most(max_sends)
+                                                                                       .at_most(max_sends).times
                                                                                        .and_wrap_original do |m, *args|
           # Always fail.
-          future = java.util.concurrent.FutureTask.new { raise "Failed" }
+          future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
           future.run
           future
         end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-kafka
 version: !ruby/object:Gem::Version
-  version: 10.2.0
+  version: 10.5.2
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-04-30 00:00:00.000000000 Z
+date: 2020-10-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement