logstash-integration-kafka 10.3.0-java → 10.5.3-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cbbffe559ff38eaa26c9922c62a3722b2b1404446602238bcc7de309ca428da6
-  data.tar.gz: 0ce7e58400e06ae5e1c7f689a14220600efe7ad9ba46f285a52323a05568f69f
+  metadata.gz: 4383db6ec7c8fa26ef358d104c490f51620f615afb2f68359b6f6e98d4e58f8b
+  data.tar.gz: 040637202d15cb1e5784104ff505f10a6610e91187a57f104a8f81cd2b24475a
 SHA512:
-  metadata.gz: 7be900607b24146853783d4e3a2e5a6d351853ab4dda0b30d10769adf8b13df21540cea2699d94660bd9e56580c9bf88e9684d5762ff7684f5d3a6804637b602
-  data.tar.gz: cd1f72db84b5af7de286d683f0f312e4b826c8d75b9b8fc475f693d8bf62b517c61950dd5d4887767f79f69df17b2c9bf1223e03880b2724333aa0b695b81525
+  metadata.gz: 98da085bceebd241a6d45f9166aa4ff1a132551cd2cda8825ceab3c22ebbb5f78579f0d1a0b596aeaf3d504147d656cf2ef784570ef3fd9ab98c792ae6e15be4
+  data.tar.gz: f552e5ec8d84f3ae7d85b3d4bc4ba4f7309321ea81efaa9bfb69a4a493141868b2576a626f0ee061bb0ebe6cbc8071993dd8592bc53030478baad2b5173e1086

data/CHANGELOG.md

@@ -1,5 +1,27 @@
+## 10.5.3
+  - Fix: set (optional) truststore when endpoint id check disabled [#60](https://github.com/logstash-plugins/logstash-integration-kafka/pull/60).
+    Since **10.1.0** disabling server host-name verification (`ssl_endpoint_identification_algorithm => ""`) did not allow 
+    the (output) plugin to set `ssl_truststore_location => "..."`.
+
+## 10.5.2
+  - Docs: explain group_id in case of multiple inputs [#59](https://github.com/logstash-plugins/logstash-integration-kafka/pull/59)
+
+## 10.5.1
+  - [DOC]Replaced plugin_header file with plugin_header-integration file. [#46](https://github.com/logstash-plugins/logstash-integration-kafka/pull/46)
+  - [DOC]Update kafka client version across kafka integration docs [#47](https://github.com/logstash-plugins/logstash-integration-kafka/pull/47)
+  - [DOC]Replace hard-coded kafka client and doc path version numbers with attributes to simplify doc maintenance [#48](https://github.com/logstash-plugins/logstash-integration-kafka/pull/48)  
+
+## 10.5.0
+  - Changed: retry sending messages only for retriable exceptions [#27](https://github.com/logstash-plugins/logstash-integration-kafka/pull/29)
+
+## 10.4.1
+  - [DOC] Fixed formatting issues and made minor content edits [#43](https://github.com/logstash-plugins/logstash-integration-kafka/pull/43)
+
+## 10.4.0
+ - added the input `isolation_level` to allow fine control of whether to return transactional messages [#44](https://github.com/logstash-plugins/logstash-integration-kafka/pull/44)
+
 ## 10.3.0
-  - added the input and output `client_dns_lookup` parameter to allow control of how DNS requests are made
+  - added the input and output `client_dns_lookup` parameter to allow control of how DNS requests are made [#28](https://github.com/logstash-plugins/logstash-integration-kafka/pull/28)
 
 ## 10.2.0
   - Changed: config defaults to be aligned with Kafka client defaults [#30](https://github.com/logstash-plugins/logstash-integration-kafka/pull/30)

data/CONTRIBUTORS

@@ -12,6 +12,7 @@ Contributors:
 * Kurt Hurtado (kurtado)
 * Ry Biesemeyer (yaauie)
 * Rob Cowart (robcowart)
+* Tim te Beek (timtebeek)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/docs/index.asciidoc

@@ -1,6 +1,7 @@
 :plugin: kafka
 :type: integration
 :no_codec:
+:kafka_client: 2.4
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!
@@ -21,11 +22,15 @@ include::{include_path}/plugin_header.asciidoc[]
 
 ==== Description
 
-The Kafka Integration Plugin provides integrated plugins for working with the https://kafka.apache.org/[Kafka] distributed streaming platform.
+The Kafka Integration Plugin provides integrated plugins for working with the
+https://kafka.apache.org/[Kafka] distributed streaming platform.
 
  - {logstash-ref}/plugins-inputs-kafka.html[Kafka Input Plugin]
  - {logstash-ref}/plugins-outputs-kafka.html[Kafka Output Plugin]
  
-This plugin uses Kafka Client 2.4. For broker compatibility, see the official https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka compatibility reference]. If the linked compatibility wiki is not up-to-date, please contact Kafka support/community to confirm compatibility.
+This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the official
+https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka
+compatibility reference]. If the linked compatibility wiki is not up-to-date,
+please contact Kafka support/community to confirm compatibility.
 
 :no_codec!:

data/docs/input-kafka.asciidoc

@@ -1,6 +1,9 @@
+:integration: kafka
 :plugin: kafka
 :type: input
 :default_codec: plain
+:kafka_client: 2.4
+:kafka_client_doc: 24
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!
@@ -17,15 +20,20 @@ END - GENERATED VARIABLES, DO NOT EDIT!
 
 === Kafka input plugin
 
-include::{include_path}/plugin_header.asciidoc[]
+include::{include_path}/plugin_header-integration.asciidoc[]
 
 ==== Description
 
 This input will read events from a Kafka topic.
 
-This plugin uses Kafka Client 2.3.0. For broker compatibility, see the official https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka compatibility reference]. If the linked compatibility wiki is not up-to-date, please contact Kafka support/community to confirm compatibility.
+This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the
+official
+https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka
+compatibility reference]. If the linked compatibility wiki is not up-to-date,
+please contact Kafka support/community to confirm compatibility.
 
-If you require features not yet available in this plugin (including client version upgrades), please file an issue with details about what you need.
+If you require features not yet available in this plugin (including client
+version upgrades), please file an issue with details about what you need.
 
 This input supports connecting to Kafka over:
 
@@ -46,9 +54,9 @@ the same `group_id`.
 Ideally you should have as many threads as the number of partitions for a perfect balance --
 more threads than partitions means that some threads will be idle
 
-For more information see https://kafka.apache.org/24/documentation.html#theconsumer
+For more information see https://kafka.apache.org/{kafka_client_doc}/documentation.html#theconsumer
 
-Kafka consumer configuration: https://kafka.apache.org/24/documentation.html#consumerconfigs
+Kafka consumer configuration: https://kafka.apache.org/{kafka_client_doc}/documentation.html#consumerconfigs
 
 ==== Metadata fields
 
@@ -59,7 +67,11 @@ The following metadata from Kafka broker are added under the `[@metadata]` field
 * `[@metadata][kafka][partition]`: Partition info for this message.
 * `[@metadata][kafka][offset]`: Original record offset for this message.
 * `[@metadata][kafka][key]`: Record key, if any.
-* `[@metadata][kafka][timestamp]`: Timestamp in the Record. Depending on your broker configuration, this can be either when the record was created (default) or when it was received by the broker. See more about property log.message.timestamp.type at https://kafka.apache.org/10/documentation.html#brokerconfigs
+* `[@metadata][kafka][timestamp]`: Timestamp in the Record. 
+Depending on your broker configuration, this can be
+either when the record was created (default) or when it was received by the
+broker. See more about property log.message.timestamp.type at
+https://kafka.apache.org/{kafka_client_doc}/documentation.html#brokerconfigs
 
 Metadata is only added to the event if the `decorate_events` option is set to true (it defaults to false).
 
@@ -73,7 +85,7 @@ This plugin supports these configuration options plus the <<plugins-{type}s-{plu
 
 NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting,
 and might change if Kafka's consumer defaults change.
-See the https://kafka.apache.org/24/documentation for more details.
+See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details.
 
 [cols="<,<,<",options="header",]
 |=======================================================================
@@ -95,6 +107,7 @@ See the https://kafka.apache.org/24/documentation for more details.
 | <<plugins-{type}s-{plugin}-fetch_min_bytes>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-group_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-heartbeat_interval_ms>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-isolation_level>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-key_deserializer_class>> |<<string,string>>|No
@@ -301,7 +314,11 @@ before answering the request.
 
 The identifier of the group this consumer belongs to. Consumer group is a single logical subscriber
 that happens to be made up of multiple processors. Messages in a topic will be distributed to all
-Logstash instances with the same `group_id`
+Logstash instances with the same `group_id`.
+
+NOTE: In cases when multiple inputs are being used in a single pipeline, reading from different topics,
+it's essential to set a different `group_id => ...` for each input. Setting a unique `client_id => ...`
+is also recommended.
 
 [id="plugins-{type}s-{plugin}-heartbeat_interval_ms"]
 ===== `heartbeat_interval_ms` 
@@ -315,6 +332,17 @@ consumers join or leave the group. The value must be set lower than
 `session.timeout.ms`, but typically should be set no higher than 1/3 of that value.
 It can be adjusted even lower to control the expected time for normal rebalances.
 
+[id="plugins-{type}s-{plugin}-isolation_level"]
+===== `isolation_level` 
+
+  * Value type is <<string,string>>
+  * Default value is `"read_uncommitted"`
+
+Controls how to read messages written transactionally. If set to `read_committed`, polling messages will only return
+transactional messages which have been committed. If set to `read_uncommitted` (the default), polling messages will
+return all messages, even transactional messages which have been aborted. Non-transactional messages will be returned
+unconditionally in either mode.
+
 [id="plugins-{type}s-{plugin}-jaas_path"]
 ===== `jaas_path` 
 
@@ -409,7 +437,7 @@ partition ownership amongst consumer instances, supported options are:
 * `sticky`
 * `cooperative_sticky`
 
-These map to Kafka's corresponding https://kafka.apache.org/24/javadoc/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.html[`ConsumerPartitionAssignor`]
+These map to Kafka's corresponding https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.html[`ConsumerPartitionAssignor`]
 implementations.
 
 [id="plugins-{type}s-{plugin}-poll_timeout_ms"]

data/docs/output-kafka.asciidoc

@@ -1,6 +1,9 @@
+:integration: kafka
 :plugin: kafka
 :type: output
 :default_codec: plain
+:kafka_client: 2.4
+:kafka_client_doc: 24
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!
@@ -17,15 +20,20 @@ END - GENERATED VARIABLES, DO NOT EDIT!
 
 === Kafka output plugin
 
-include::{include_path}/plugin_header.asciidoc[]
+include::{include_path}/plugin_header-integration.asciidoc[]
 
 ==== Description
 
 Write events to a Kafka topic. 
 
-This plugin uses Kafka Client 2.3.0. For broker compatibility, see the official https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka compatibility reference]. If the linked compatibility wiki is not up-to-date, please contact Kafka support/community to confirm compatibility.
+This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the
+official
+https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka
+compatibility reference]. If the linked compatibility wiki is not up-to-date,
+please contact Kafka support/community to confirm compatibility.
 
-If you require features not yet available in this plugin (including client version upgrades), please file an issue with details about what you need.
+If you require features not yet available in this plugin (including client
+version upgrades), please file an issue with details about what you need.
 
 This output supports connecting to Kafka over:
 
@@ -36,9 +44,12 @@ By default security is disabled but can be turned on as needed.
 
 The only required configuration is the topic_id. 
 
-The default codec is plain. Logstash will encode your events with not only the message field but also with a timestamp and hostname.
+The default codec is plain. Logstash will encode your events with not only the
+message field but also with a timestamp and hostname.
+
+If you want the full content of your events to be sent as json, you should set
+the codec in the output configuration like this:
 
-If you want the full content of your events to be sent as json, you should set the codec in the output configuration like this:
 [source,ruby]
     output {
       kafka {
@@ -47,9 +58,11 @@ If you want the full content of your events to be sent as json, you should set t
       }
     }
     
-For more information see https://kafka.apache.org/24/documentation.html#theproducer
+For more information see
+https://kafka.apache.org/{kafka_client_doc}/documentation.html#theproducer
 
-Kafka producer configuration: https://kafka.apache.org/24/documentation.html#producerconfigs
+Kafka producer configuration:
+https://kafka.apache.org/{kafka_client_doc}/documentation.html#producerconfigs
 
 [id="plugins-{type}s-{plugin}-options"]
 ==== Kafka Output Configuration Options
@@ -58,7 +71,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 
 NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting,
 and might change if Kafka's producer defaults change.
-See the https://kafka.apache.org/24/documentation for more details.
+See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details.
 
 [cols="<,<,<",options="header",]
 |=======================================================================
@@ -115,10 +128,13 @@ output plugins.
 The number of acknowledgments the producer requires the leader to have received
 before considering a request complete.
 
-acks=0,   the producer will not wait for any acknowledgment from the server at all.
-acks=1,   This will mean the leader will write the record to its local log but
-          will respond without awaiting full acknowledgement from all followers.
-acks=all, This means the leader will wait for the full set of in-sync replicas to acknowledge the record.
+`acks=0`. The producer will not wait for any acknowledgment from the server.
+
+`acks=1`. The leader will write the record to its local log, but will respond
+without waiting for full acknowledgement from all followers.
+          
+`acks=all`. The leader will wait for the full set of in-sync replicas before
+acknowledging the record.
 
 [id="plugins-{type}s-{plugin}-batch_size"]
 ===== `batch_size` 
@@ -154,11 +170,12 @@ The total bytes of memory the producer can use to buffer records waiting to be s
 ===== `client_dns_lookup` 
 
   * Value type is <<string,string>>
+  * Valid options are `use_all_dns_ips`, `resolve_canonical_bootstrap_servers_only`, `default`
   * Default value is `"default"`
 
-How DNS lookups should be done. If set to `use_all_dns_ips`, when the lookup returns multiple 
-IP addresses for a hostname, they will all be attempted to connect to before failing the 
-connection. If the value is `resolve_canonical_bootstrap_servers_only` each entry will be 
+Controls how DNS lookups are done. If set to `use_all_dns_ips`, Logstash tries
+all IP addresses returned for a hostname before failing the connection.
+If set to `resolve_canonical_bootstrap_servers_only`, each entry will be 
 resolved and expanded into a list of canonical names.
 
 [id="plugins-{type}s-{plugin}-client_id"]
@@ -178,7 +195,7 @@ ip/port by allowing a logical application name to be included with the request
   * Default value is `"none"`
 
 The compression type for all data generated by the producer.
-The default is none (i.e. no compression). Valid values are none, gzip, or snappy.
+The default is none (i.e. no compression). Valid values are none, gzip, snappy, or lz4.
 
 [id="plugins-{type}s-{plugin}-jaas_path"]
 ===== `jaas_path` 
@@ -323,6 +340,15 @@ Kafka down, etc).
 
 A value less than zero is a configuration error.
 
+Starting with version 10.5.0, this plugin will only retry exceptions that are a subclass of
+https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/RetriableException.html[RetriableException]
+and
+https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/InterruptException.html[InterruptException].
+If producing a message throws any other exception, an error is logged and the message is dropped without retrying.
+This prevents the Logstash pipeline from hanging indefinitely.
+
+In versions prior to 10.5.0, any exception is retried indefinitely unless the `retries` option is configured.
+
 [id="plugins-{type}s-{plugin}-retry_backoff_ms"]
 ===== `retry_backoff_ms` 
 

data/lib/logstash/inputs/kafka.rb

@@ -3,6 +3,7 @@ require 'logstash/inputs/base'
 require 'stud/interval'
 require 'java'
 require 'logstash-integration-kafka_jars.rb'
+require 'logstash/plugin_mixins/kafka_support'
 
 # This input will read events from a Kafka topic. It uses the 0.10 version of
 # the consumer API provided by Kafka to read messages from the broker.
@@ -48,6 +49,9 @@ require 'logstash-integration-kafka_jars.rb'
 # Kafka consumer configuration: http://kafka.apache.org/documentation.html#consumerconfigs
 #
 class LogStash::Inputs::Kafka < LogStash::Inputs::Base
+
+  include LogStash::PluginMixins::KafkaSupport
+
   config_name 'kafka'
 
   default :codec, 'plain'
@@ -114,6 +118,11 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   # `session.timeout.ms`, but typically should be set no higher than 1/3 of that value.
   # It can be adjusted even lower to control the expected time for normal rebalances.
   config :heartbeat_interval_ms, :validate => :number, :default => 3000 # Kafka default
+  # Controls how to read messages written transactionally. If set to read_committed, consumer.poll()
+  # will only return transactional messages which have been committed. If set to read_uncommitted'
+  # (the default), consumer.poll() will return all messages, even transactional messages which have
+  # been aborted. Non-transactional messages will be returned unconditionally in either mode.
+  config :isolation_level, :validate => ["read_uncommitted", "read_committed"], :default => "read_uncommitted" # Kafka default
   # Java Class used to deserialize the record's key
   config :key_deserializer_class, :validate => :string, :default => "org.apache.kafka.common.serialization.StringDeserializer"
   # The maximum delay between invocations of poll() when using consumer group management. This places 
@@ -311,6 +320,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
       props.put(kafka::FETCH_MIN_BYTES_CONFIG, fetch_min_bytes.to_s) unless fetch_min_bytes.nil?
       props.put(kafka::GROUP_ID_CONFIG, group_id)
       props.put(kafka::HEARTBEAT_INTERVAL_MS_CONFIG, heartbeat_interval_ms.to_s) unless heartbeat_interval_ms.nil?
+      props.put(kafka::ISOLATION_LEVEL_CONFIG, isolation_level)
       props.put(kafka::KEY_DESERIALIZER_CLASS_CONFIG, key_deserializer_class)
       props.put(kafka::MAX_PARTITION_FETCH_BYTES_CONFIG, max_partition_fetch_bytes.to_s) unless max_partition_fetch_bytes.nil?
       props.put(kafka::MAX_POLL_RECORDS_CONFIG, max_poll_records.to_s) unless max_poll_records.nil?
@@ -364,29 +374,4 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
     end
   end
 
-  def set_trustore_keystore_config(props)
-    props.put("ssl.truststore.type", ssl_truststore_type) unless ssl_truststore_type.nil?
-    props.put("ssl.truststore.location", ssl_truststore_location) unless ssl_truststore_location.nil?
-    props.put("ssl.truststore.password", ssl_truststore_password.value) unless ssl_truststore_password.nil?
-
-    # Client auth stuff
-    props.put("ssl.keystore.type", ssl_keystore_type) unless ssl_keystore_type.nil?
-    props.put("ssl.key.password", ssl_key_password.value) unless ssl_key_password.nil?
-    props.put("ssl.keystore.location", ssl_keystore_location) unless ssl_keystore_location.nil?
-    props.put("ssl.keystore.password", ssl_keystore_password.value) unless ssl_keystore_password.nil?
-    props.put("ssl.endpoint.identification.algorithm", ssl_endpoint_identification_algorithm) unless ssl_endpoint_identification_algorithm.nil?
-  end
-
-  def set_sasl_config(props)
-    java.lang.System.setProperty("java.security.auth.login.config", jaas_path) unless jaas_path.nil?
-    java.lang.System.setProperty("java.security.krb5.conf", kerberos_config) unless kerberos_config.nil?
-
-    props.put("sasl.mechanism", sasl_mechanism)
-    if sasl_mechanism == "GSSAPI" && sasl_kerberos_service_name.nil?
-      raise LogStash::ConfigurationError, "sasl_kerberos_service_name must be specified when SASL mechanism is GSSAPI"
-    end
-
-    props.put("sasl.kerberos.service.name", sasl_kerberos_service_name) unless sasl_kerberos_service_name.nil?
-    props.put("sasl.jaas.config", sasl_jaas_config) unless sasl_jaas_config.nil?
-  end
 end #class LogStash::Inputs::Kafka

data/lib/logstash/outputs/kafka.rb

@@ -2,6 +2,7 @@ require 'logstash/namespace'
 require 'logstash/outputs/base'
 require 'java'
 require 'logstash-integration-kafka_jars.rb'
+require 'logstash/plugin_mixins/kafka_support'
 
 # Write events to a Kafka topic. This uses the Kafka Producer API to write messages to a topic on
 # the broker.
@@ -50,6 +51,8 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
 
   java_import org.apache.kafka.clients.producer.ProducerRecord
 
+  include LogStash::PluginMixins::KafkaSupport
+
   declare_threadsafe!
 
   config_name 'kafka'
@@ -236,7 +239,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
     remaining = @retries
 
     while batch.any?
-      if !remaining.nil?
+      unless remaining.nil?
         if remaining < 0
           # TODO(sissel): Offer to DLQ? Then again, if it's a transient fault,
           # DLQing would make things worse (you dlq data that would be successful
@@ -255,27 +258,39 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
         begin
           # send() can throw an exception even before the future is created.
           @producer.send(record)
-        rescue org.apache.kafka.common.errors.TimeoutException => e
-          failures << record
-          nil
-        rescue org.apache.kafka.common.errors.InterruptException => e
+        rescue org.apache.kafka.common.errors.InterruptException,
+               org.apache.kafka.common.errors.RetriableException => e
+          logger.info("producer send failed, will retry sending", :exception => e.class, :message => e.message)
           failures << record
           nil
-        rescue org.apache.kafka.common.errors.SerializationException => e
-          # TODO(sissel): Retrying will fail because the data itself has a problem serializing.
-          # TODO(sissel): Let's add DLQ here.
-          failures << record
+        rescue org.apache.kafka.common.KafkaException => e
+          # This error is not retriable, drop event
+          # TODO: add DLQ support
+          logger.warn("producer send failed, dropping record",:exception => e.class, :message => e.message,
+                      :record_value => record.value)
           nil
         end
-      end.compact
+      end
 
       futures.each_with_index do |future, i|
-        begin
-          result = future.get()
-        rescue => e
-          # TODO(sissel): Add metric to count failures, possibly by exception type.
-          logger.warn("producer send failed", :exception => e.class, :message => e.message)
-          failures << batch[i]
+        # We cannot skip nils using `futures.compact` because then our index `i` will not align with `batch`
+        unless future.nil?
+          begin
+            future.get
+          rescue java.util.concurrent.ExecutionException => e
+            # TODO(sissel): Add metric to count failures, possibly by exception type.
+            if e.get_cause.is_a? org.apache.kafka.common.errors.RetriableException or
+               e.get_cause.is_a? org.apache.kafka.common.errors.InterruptException
+              logger.info("producer send failed, will retry sending", :exception => e.cause.class,
+                          :message => e.cause.message)
+              failures << batch[i]
+            elsif e.get_cause.is_a? org.apache.kafka.common.KafkaException
+              # This error is not retriable, drop event
+              # TODO: add DLQ support
+              logger.warn("producer send failed, dropping record", :exception => e.cause.class,
+                          :message => e.cause.message, :record_value => batch[i].value)
+            end
+          end
         end
       end
 
@@ -377,35 +392,4 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
     end
   end
 
-  def set_trustore_keystore_config(props)
-    unless ssl_endpoint_identification_algorithm.to_s.strip.empty?
-      if ssl_truststore_location.nil?
-        raise LogStash::ConfigurationError, "ssl_truststore_location must be set when SSL is enabled"
-      end
-      props.put("ssl.truststore.type", ssl_truststore_type) unless ssl_truststore_type.nil?
-      props.put("ssl.truststore.location", ssl_truststore_location)
-      props.put("ssl.truststore.password", ssl_truststore_password.value) unless ssl_truststore_password.nil?
-    end
-
-    # Client auth stuff
-    props.put("ssl.keystore.type", ssl_keystore_type) unless ssl_keystore_type.nil?
-    props.put("ssl.key.password", ssl_key_password.value) unless ssl_key_password.nil?
-    props.put("ssl.keystore.location", ssl_keystore_location) unless ssl_keystore_location.nil?
-    props.put("ssl.keystore.password", ssl_keystore_password.value) unless ssl_keystore_password.nil?
-    props.put("ssl.endpoint.identification.algorithm", ssl_endpoint_identification_algorithm) unless ssl_endpoint_identification_algorithm.nil?
-  end
-
-  def set_sasl_config(props)
-    java.lang.System.setProperty("java.security.auth.login.config", jaas_path) unless jaas_path.nil?
-    java.lang.System.setProperty("java.security.krb5.conf", kerberos_config) unless kerberos_config.nil?
-
-    props.put("sasl.mechanism",sasl_mechanism)
-    if sasl_mechanism == "GSSAPI" && sasl_kerberos_service_name.nil?
-      raise LogStash::ConfigurationError, "sasl_kerberos_service_name must be specified when SASL mechanism is GSSAPI"
-    end
-
-    props.put("sasl.kerberos.service.name", sasl_kerberos_service_name) unless sasl_kerberos_service_name.nil?
-    props.put("sasl.jaas.config", sasl_jaas_config) unless sasl_jaas_config.nil?
-  end
-
 end #class LogStash::Outputs::Kafka

data/lib/logstash/plugin_mixins/kafka_support.rb

@@ -0,0 +1,29 @@
+module LogStash module PluginMixins module KafkaSupport
+
+  def set_trustore_keystore_config(props)
+    props.put("ssl.truststore.type", ssl_truststore_type) unless ssl_truststore_type.nil?
+    props.put("ssl.truststore.location", ssl_truststore_location) unless ssl_truststore_location.nil?
+    props.put("ssl.truststore.password", ssl_truststore_password.value) unless ssl_truststore_password.nil?
+
+    # Client auth stuff
+    props.put("ssl.keystore.type", ssl_keystore_type) unless ssl_keystore_type.nil?
+    props.put("ssl.key.password", ssl_key_password.value) unless ssl_key_password.nil?
+    props.put("ssl.keystore.location", ssl_keystore_location) unless ssl_keystore_location.nil?
+    props.put("ssl.keystore.password", ssl_keystore_password.value) unless ssl_keystore_password.nil?
+    props.put("ssl.endpoint.identification.algorithm", ssl_endpoint_identification_algorithm) unless ssl_endpoint_identification_algorithm.nil?
+  end
+
+  def set_sasl_config(props)
+    java.lang.System.setProperty("java.security.auth.login.config", jaas_path) unless jaas_path.nil?
+    java.lang.System.setProperty("java.security.krb5.conf", kerberos_config) unless kerberos_config.nil?
+
+    props.put("sasl.mechanism", sasl_mechanism)
+    if sasl_mechanism == "GSSAPI" && sasl_kerberos_service_name.nil?
+      raise LogStash::ConfigurationError, "sasl_kerberos_service_name must be specified when SASL mechanism is GSSAPI"
+    end
+
+    props.put("sasl.kerberos.service.name", sasl_kerberos_service_name) unless sasl_kerberos_service_name.nil?
+    props.put("sasl.jaas.config", sasl_jaas_config) unless sasl_jaas_config.nil?
+  end
+
+end end end

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '10.3.0'
+  s.version         = '10.5.3'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+

data/spec/unit/outputs/kafka_spec.rb

@@ -50,20 +50,22 @@ describe "outputs/kafka" do
       kafka.multi_receive([event])
     end
 
-    it 'should raise config error when truststore location is not set and ssl is enabled' do
+    it 'should not raise config error when truststore location is not set and ssl is enabled' do
       kafka = LogStash::Outputs::Kafka.new(simple_kafka_config.merge("security_protocol" => "SSL"))
-      expect { kafka.register }.to raise_error(LogStash::ConfigurationError, /ssl_truststore_location must be set when SSL is enabled/)
+      expect(org.apache.kafka.clients.producer.KafkaProducer).to receive(:new)
+      expect { kafka.register }.to_not raise_error
     end
   end
 
-  context "when KafkaProducer#send() raises an exception" do
+  context "when KafkaProducer#send() raises a retriable exception" do
     let(:failcount) { (rand * 10).to_i }
     let(:sendcount) { failcount + 1 }
 
     let(:exception_classes) { [
       org.apache.kafka.common.errors.TimeoutException,
+      org.apache.kafka.common.errors.DisconnectException,
+      org.apache.kafka.common.errors.CoordinatorNotAvailableException,
       org.apache.kafka.common.errors.InterruptException,
-      org.apache.kafka.common.errors.SerializationException
     ] }
 
     before do
@@ -88,6 +90,37 @@ describe "outputs/kafka" do
     end
   end
 
+  context "when KafkaProducer#send() raises a non-retriable exception" do
+    let(:failcount) { (rand * 10).to_i }
+
+    let(:exception_classes) { [
+        org.apache.kafka.common.errors.SerializationException,
+        org.apache.kafka.common.errors.RecordTooLargeException,
+        org.apache.kafka.common.errors.InvalidTopicException
+    ] }
+
+    before do
+      count = 0
+      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
+        .exactly(1).times
+        .and_wrap_original do |m, *args|
+        if count < failcount # fail 'failcount' times in a row.
+          count += 1
+          # Pick an exception at random
+          raise exception_classes.shuffle.first.new("injected exception for testing")
+        else
+          m.call(*args) # call original
+        end
+      end
+    end
+
+    it "should not retry" do
+      kafka = LogStash::Outputs::Kafka.new(simple_kafka_config)
+      kafka.register
+      kafka.multi_receive([event])
+    end
+  end
+
   context "when a send fails" do
     context "and the default retries behavior is used" do
       # Fail this many times and then finally succeed.
@@ -107,7 +140,7 @@ describe "outputs/kafka" do
             # inject some failures.
 
             # Return a custom Future that will raise an exception to simulate a Kafka send() problem.
-            future = java.util.concurrent.FutureTask.new { raise "Failed" }
+            future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
             future.run
             future
           else
@@ -129,7 +162,7 @@ describe "outputs/kafka" do
                                                                                        .once
                                                                                        .and_wrap_original do |m, *args|
           # Always fail.
-          future = java.util.concurrent.FutureTask.new { raise "Failed" }
+          future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
           future.run
           future
         end
@@ -143,7 +176,7 @@ describe "outputs/kafka" do
                                                                                        .once
                                                                                        .and_wrap_original do |m, *args|
           # Always fail.
-          future = java.util.concurrent.FutureTask.new { raise "Failed" }
+          future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
           future.run
           future
         end
@@ -164,7 +197,7 @@ describe "outputs/kafka" do
               .at_most(max_sends).times
               .and_wrap_original do |m, *args|
           # Always fail.
-          future = java.util.concurrent.FutureTask.new { raise "Failed" }
+          future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
           future.run
           future
         end
@@ -175,10 +208,10 @@ describe "outputs/kafka" do
 
       it 'should only sleep retries number of times' do
         expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
-                                                                                       .at_most(max_sends)
+                                                                                       .at_most(max_sends).times
                                                                                        .and_wrap_original do |m, *args|
           # Always fail.
-          future = java.util.concurrent.FutureTask.new { raise "Failed" }
+          future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
           future.run
           future
         end
@@ -193,21 +226,31 @@ describe "outputs/kafka" do
   context 'when ssl endpoint identification disabled' do
 
     let(:config) do
-      simple_kafka_config.merge('ssl_endpoint_identification_algorithm' => '', 'security_protocol' => 'SSL')
+      simple_kafka_config.merge(
+          'security_protocol' => 'SSL',
+          'ssl_endpoint_identification_algorithm' => '',
+          'ssl_truststore_location' => truststore_path,
+      )
+    end
+
+    let(:truststore_path) do
+      File.join(File.dirname(__FILE__), '../../fixtures/trust-store_stub.jks')
     end
 
     subject { LogStash::Outputs::Kafka.new(config) }
 
-    it 'does not configure truststore' do
+    it 'sets empty ssl.endpoint.identification.algorithm' do
       expect(org.apache.kafka.clients.producer.KafkaProducer).
-          to receive(:new).with(hash_excluding('ssl.truststore.location' => anything))
+          to receive(:new).with(hash_including('ssl.endpoint.identification.algorithm' => ''))
       subject.register
     end
 
-    it 'sets empty ssl.endpoint.identification.algorithm' do
+    it 'configures truststore' do
       expect(org.apache.kafka.clients.producer.KafkaProducer).
-          to receive(:new).with(hash_including('ssl.endpoint.identification.algorithm' => ''))
+          to receive(:new).with(hash_including('ssl.truststore.location' => truststore_path))
       subject.register
     end
+
   end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-kafka
 version: !ruby/object:Gem::Version
-  version: 10.3.0
+  version: 10.5.3
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-06-18 00:00:00.000000000 Z
+date: 2020-10-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -183,7 +183,9 @@ files:
 - lib/logstash-integration-kafka_jars.rb
 - lib/logstash/inputs/kafka.rb
 - lib/logstash/outputs/kafka.rb
+- lib/logstash/plugin_mixins/kafka_support.rb
 - logstash-integration-kafka.gemspec
+- spec/fixtures/trust-store_stub.jks
 - spec/integration/inputs/kafka_spec.rb
 - spec/integration/outputs/kafka_spec.rb
 - spec/unit/inputs/kafka_spec.rb
@@ -222,6 +224,7 @@ signing_key:
 specification_version: 4
 summary: Integration with Kafka - input and output plugins
 test_files:
+- spec/fixtures/trust-store_stub.jks
 - spec/integration/inputs/kafka_spec.rb
 - spec/integration/outputs/kafka_spec.rb
 - spec/unit/inputs/kafka_spec.rb