logstash-integration-kafka 10.1.0-java → 10.5.1-java

data/docs/output-kafka.asciidoc

@@ -1,6 +1,9 @@
+:integration: kafka
 :plugin: kafka
 :type: output
 :default_codec: plain
+:kafka_client: 2.4
+:kafka_client_doc: 24
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!
@@ -17,15 +20,20 @@ END - GENERATED VARIABLES, DO NOT EDIT!
 
 === Kafka output plugin
 
-include::{include_path}/plugin_header.asciidoc[]
+include::{include_path}/plugin_header-integration.asciidoc[]
 
 ==== Description
 
 Write events to a Kafka topic. 
 
-This plugin uses Kafka Client 2.1.0. For broker compatibility, see the official https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka compatibility reference]. If the linked compatibility wiki is not up-to-date, please contact Kafka support/community to confirm compatibility.
+This plugin uses Kafka Client {kafka_client}. For broker compatibility, see the
+official
+https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka
+compatibility reference]. If the linked compatibility wiki is not up-to-date,
+please contact Kafka support/community to confirm compatibility.
 
-If you require features not yet available in this plugin (including client version upgrades), please file an issue with details about what you need.
+If you require features not yet available in this plugin (including client
+version upgrades), please file an issue with details about what you need.
 
 This output supports connecting to Kafka over:
 
@@ -36,9 +44,12 @@ By default security is disabled but can be turned on as needed.
 
 The only required configuration is the topic_id. 
 
-The default codec is plain. Logstash will encode your events with not only the message field but also with a timestamp and hostname.
+The default codec is plain. Logstash will encode your events with not only the
+message field but also with a timestamp and hostname.
+
+If you want the full content of your events to be sent as json, you should set
+the codec in the output configuration like this:
 
-If you want the full content of your events to be sent as json, you should set the codec in the output configuration like this:
 [source,ruby]
     output {
       kafka {
@@ -47,15 +58,21 @@ If you want the full content of your events to be sent as json, you should set t
       }
     }
     
-For more information see http://kafka.apache.org/documentation.html#theproducer
+For more information see
+https://kafka.apache.org/{kafka_client_doc}/documentation.html#theproducer
 
-Kafka producer configuration: http://kafka.apache.org/documentation.html#newproducerconfigs
+Kafka producer configuration:
+https://kafka.apache.org/{kafka_client_doc}/documentation.html#producerconfigs
 
 [id="plugins-{type}s-{plugin}-options"]
 ==== Kafka Output Configuration Options
 
 This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
 
+NOTE: Some of these options map to a Kafka option. Defaults usually reflect the Kafka default setting,
+and might change if Kafka's producer defaults change.
+See the https://kafka.apache.org/{kafka_client_doc}/documentation for more details.
+
 [cols="<,<,<",options="header",]
 |=======================================================================
 |Setting |Input type|Required
@@ -63,6 +80,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-batch_size>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-bootstrap_servers>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-buffer_memory>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-client_dns_lookup>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-client_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-compression_type>> |<<string,string>>, one of `["none", "gzip", "snappy", "lz4"]`|No
 | <<plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No
@@ -76,7 +94,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-partitioner>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-receive_buffer_bytes>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-reconnect_backoff_ms>> |<<number,number>>|No
-| <<plugins-{type}s-{plugin}-request_timeout_ms>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-request_timeout_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-retries>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-retry_backoff_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-sasl_jaas_config>> |<<string,string>>|No
@@ -110,16 +128,19 @@ output plugins.
 The number of acknowledgments the producer requires the leader to have received
 before considering a request complete.
 
-acks=0,   the producer will not wait for any acknowledgment from the server at all.
-acks=1,   This will mean the leader will write the record to its local log but
-          will respond without awaiting full acknowledgement from all followers.
-acks=all, This means the leader will wait for the full set of in-sync replicas to acknowledge the record.
+`acks=0`. The producer will not wait for any acknowledgment from the server.
+
+`acks=1`. The leader will write the record to its local log, but will respond
+without waiting for full acknowledgement from all followers.
+          
+`acks=all`. The leader will wait for the full set of in-sync replicas before
+acknowledging the record.
 
 [id="plugins-{type}s-{plugin}-batch_size"]
 ===== `batch_size` 
 
   * Value type is <<number,number>>
-  * Default value is `16384`
+  * Default value is `16384`.
 
 The producer will attempt to batch records together into fewer requests whenever multiple
 records are being sent to the same partition. This helps performance on both the client
@@ -141,10 +162,22 @@ subset of brokers.
 ===== `buffer_memory` 
 
   * Value type is <<number,number>>
-  * Default value is `33554432`
+  * Default value is `33554432` (32MB).
 
 The total bytes of memory the producer can use to buffer records waiting to be sent to the server.
 
+[id="plugins-{type}s-{plugin}-client_dns_lookup"]
+===== `client_dns_lookup` 
+
+  * Value type is <<string,string>>
+  * Valid options are `use_all_dns_ips`, `resolve_canonical_bootstrap_servers_only`, `default`
+  * Default value is `"default"`
+
+Controls how DNS lookups are done. If set to `use_all_dns_ips`, Logstash tries
+all IP addresses returned for a hostname before failing the connection.
+If set to `resolve_canonical_bootstrap_servers_only`, each entry will be 
+resolved and expanded into a list of canonical names.
+
 [id="plugins-{type}s-{plugin}-client_id"]
 ===== `client_id` 
 
@@ -162,7 +195,7 @@ ip/port by allowing a logical application name to be included with the request
   * Default value is `"none"`
 
 The compression type for all data generated by the producer.
-The default is none (i.e. no compression). Valid values are none, gzip, or snappy.
+The default is none (i.e. no compression). Valid values are none, gzip, snappy, or lz4.
 
 [id="plugins-{type}s-{plugin}-jaas_path"]
 ===== `jaas_path` 
@@ -221,7 +254,7 @@ to allow other records to be sent so that the sends can be batched together.
 ===== `max_request_size` 
 
   * Value type is <<number,number>>
-  * Default value is `1048576`
+  * Default value is `1048576` (1MB).
 
 The maximum size of a request
 
@@ -231,23 +264,23 @@ The maximum size of a request
   * Value type is <<string,string>>
   * There is no default value for this setting.
 
-The key for the message
+The key for the message.
 
 [id="plugins-{type}s-{plugin}-metadata_fetch_timeout_ms"]
 ===== `metadata_fetch_timeout_ms` 
 
   * Value type is <<number,number>>
-  * Default value is `60000`
+  * Default value is `60000` milliseconds (60 seconds).
 
-the timeout setting for initial metadata request to fetch topic metadata.
+The timeout setting for initial metadata request to fetch topic metadata.
 
 [id="plugins-{type}s-{plugin}-metadata_max_age_ms"]
 ===== `metadata_max_age_ms` 
 
   * Value type is <<number,number>>
-  * Default value is `300000`
+  * Default value is `300000` milliseconds (5 minutes).
 
-the max time in milliseconds before a metadata refresh is forced.
+The max time in milliseconds before a metadata refresh is forced.
 
 [id="plugins-{type}s-{plugin}-partitioner"]
 ===== `partitioner`
@@ -268,7 +301,7 @@ Available options for choosing a partitioning strategy are as follows:
 ===== `receive_buffer_bytes` 
 
   * Value type is <<number,number>>
-  * Default value is `32768`
+  * Default value is `32768` (32KB).
 
 The size of the TCP receive buffer to use when reading data
 
@@ -276,15 +309,15 @@ The size of the TCP receive buffer to use when reading data
 ===== `reconnect_backoff_ms` 
 
   * Value type is <<number,number>>
-  * Default value is `10`
+  * Default value is `50`.
 
 The amount of time to wait before attempting to reconnect to a given host when a connection fails.
 
 [id="plugins-{type}s-{plugin}-request_timeout_ms"]
 ===== `request_timeout_ms` 
 
-  * Value type is <<string,string>>
-  * There is no default value for this setting.
+  * Value type is <<number,number>>
+  * Default value is `40000` milliseconds (40 seconds).
 
 The configuration controls the maximum amount of time the client will wait
 for the response of a request. If the response is not received before the timeout
@@ -307,11 +340,20 @@ Kafka down, etc).
 
 A value less than zero is a configuration error.
 
+Starting with version 10.5.0, this plugin will only retry exceptions that are a subclass of
+https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/RetriableException.html[RetriableException]
+and
+https://kafka.apache.org/{kafka_client_doc}/javadoc/org/apache/kafka/common/errors/InterruptException.html[InterruptException].
+If producing a message throws any other exception, an error is logged and the message is dropped without retrying.
+This prevents the Logstash pipeline from hanging indefinitely.
+
+In versions prior to 10.5.0, any exception is retried indefinitely unless the `retries` option is configured.
+
 [id="plugins-{type}s-{plugin}-retry_backoff_ms"]
 ===== `retry_backoff_ms` 
 
   * Value type is <<number,number>>
-  * Default value is `100`
+  * Default value is `100` milliseconds.
 
 The amount of time to wait before attempting to retry a failed produce request to a given topic partition.
 
@@ -364,7 +406,7 @@ Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SA
 ===== `send_buffer_bytes` 
 
   * Value type is <<number,number>>
-  * Default value is `131072`
+  * Default value is `131072` (128KB).
 
 The size of the TCP send buffer to use when sending data.
 

data/lib/logstash/inputs/kafka.rb

@@ -53,7 +53,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   default :codec, 'plain'
 
   # The frequency in milliseconds that the consumer offsets are committed to Kafka.
-  config :auto_commit_interval_ms, :validate => :string, :default => "5000"
+  config :auto_commit_interval_ms, :validate => :number, :default => 5000 # Kafka default
   # What to do when there is no initial offset in Kafka or if an offset is out of range:
   #
   # * earliest: automatically reset the offset to the earliest offset
@@ -70,35 +70,40 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   # Automatically check the CRC32 of the records consumed. This ensures no on-the-wire or on-disk
   # corruption to the messages occurred. This check adds some overhead, so it may be
   # disabled in cases seeking extreme performance.
-  config :check_crcs, :validate => :string
+  config :check_crcs, :validate => :boolean, :default => true
+  # How DNS lookups should be done. If set to `use_all_dns_ips`, when the lookup returns multiple 
+  # IP addresses for a hostname, they will all be attempted to connect to before failing the 
+  # connection. If the value is `resolve_canonical_bootstrap_servers_only` each entry will be 
+  # resolved and expanded into a list of canonical names.
+  config :client_dns_lookup, :validate => ["default", "use_all_dns_ips", "resolve_canonical_bootstrap_servers_only"], :default => "default"
   # The id string to pass to the server when making requests. The purpose of this
   # is to be able to track the source of requests beyond just ip/port by allowing
   # a logical application name to be included.
   config :client_id, :validate => :string, :default => "logstash"
   # Close idle connections after the number of milliseconds specified by this config.
-  config :connections_max_idle_ms, :validate => :string
+  config :connections_max_idle_ms, :validate => :number, :default => 540_000 # (9m) Kafka default
   # Ideally you should have as many threads as the number of partitions for a perfect
   # balance — more threads than partitions means that some threads will be idle
   config :consumer_threads, :validate => :number, :default => 1
   # If true, periodically commit to Kafka the offsets of messages already returned by the consumer. 
   # This committed offset will be used when the process fails as the position from
   # which the consumption will begin.
-  config :enable_auto_commit, :validate => :string, :default => "true"
+  config :enable_auto_commit, :validate => :boolean, :default => true
   # Whether records from internal topics (such as offsets) should be exposed to the consumer.
   # If set to true the only way to receive records from an internal topic is subscribing to it.
   config :exclude_internal_topics, :validate => :string
   # The maximum amount of data the server should return for a fetch request. This is not an 
   # absolute maximum, if the first message in the first non-empty partition of the fetch is larger 
   # than this value, the message will still be returned to ensure that the consumer can make progress.
-  config :fetch_max_bytes, :validate => :string
+  config :fetch_max_bytes, :validate => :number, :default => 52_428_800 # (50MB) Kafka default
   # The maximum amount of time the server will block before answering the fetch request if
   # there isn't sufficient data to immediately satisfy `fetch_min_bytes`. This
   # should be less than or equal to the timeout used in `poll_timeout_ms`
-  config :fetch_max_wait_ms, :validate => :string
+  config :fetch_max_wait_ms, :validate => :number, :default => 500 # Kafka default
   # The minimum amount of data the server should return for a fetch request. If insufficient
   # data is available the request will wait for that much data to accumulate
   # before answering the request.
-  config :fetch_min_bytes, :validate => :string
+  config :fetch_min_bytes, :validate => :number
   # The identifier of the group this consumer belongs to. Consumer group is a single logical subscriber
   # that happens to be made up of multiple processors. Messages in a topic will be distributed to all
   # Logstash instances with the same `group_id`
@@ -108,50 +113,55 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   # consumers join or leave the group. The value must be set lower than
   # `session.timeout.ms`, but typically should be set no higher than 1/3 of that value.
   # It can be adjusted even lower to control the expected time for normal rebalances.
-  config :heartbeat_interval_ms, :validate => :string
+  config :heartbeat_interval_ms, :validate => :number, :default => 3000 # Kafka default
+  # Controls how to read messages written transactionally. If set to read_committed, consumer.poll()
+  # will only return transactional messages which have been committed. If set to read_uncommitted'
+  # (the default), consumer.poll() will return all messages, even transactional messages which have
+  # been aborted. Non-transactional messages will be returned unconditionally in either mode.
+  config :isolation_level, :validate => ["read_uncommitted", "read_committed"], :default => "read_uncommitted" # Kafka default
   # Java Class used to deserialize the record's key
   config :key_deserializer_class, :validate => :string, :default => "org.apache.kafka.common.serialization.StringDeserializer"
   # The maximum delay between invocations of poll() when using consumer group management. This places 
   # an upper bound on the amount of time that the consumer can be idle before fetching more records. 
   # If poll() is not called before expiration of this timeout, then the consumer is considered failed and 
   # the group will rebalance in order to reassign the partitions to another member.
-  # The value of the configuration `request_timeout_ms` must always be larger than max_poll_interval_ms
-  config :max_poll_interval_ms, :validate => :string
+  config :max_poll_interval_ms, :validate => :number, :default => 300_000 # (5m) Kafka default
   # The maximum amount of data per-partition the server will return. The maximum total memory used for a
   # request will be <code>#partitions * max.partition.fetch.bytes</code>. This size must be at least
   # as large as the maximum message size the server allows or else it is possible for the producer to
   # send messages larger than the consumer can fetch. If that happens, the consumer can get stuck trying
   # to fetch a large message on a certain partition.
-  config :max_partition_fetch_bytes, :validate => :string
+  config :max_partition_fetch_bytes, :validate => :number, :default => 1_048_576 # (1MB) Kafka default
   # The maximum number of records returned in a single call to poll().
-  config :max_poll_records, :validate => :string
+  config :max_poll_records, :validate => :number, :default => 500 # Kafka default
   # The period of time in milliseconds after which we force a refresh of metadata even if
   # we haven't seen any partition leadership changes to proactively discover any new brokers or partitions
-  config :metadata_max_age_ms, :validate => :string
+  config :metadata_max_age_ms, :validate => :number, :default => 300_000 # (5m) Kafka default
   # The name of the partition assignment strategy that the client uses to distribute
   # partition ownership amongst consumer instances, supported options are `range`,
   # `round_robin`, `sticky` and `cooperative_sticky`
   # (for backwards compatibility setting the class name directly is supported).
   config :partition_assignment_strategy, :validate => :string
   # The size of the TCP receive buffer (SO_RCVBUF) to use when reading data.
-  config :receive_buffer_bytes, :validate => :string
-  # The amount of time to wait before attempting to reconnect to a given host.
+  # If the value is `-1`, the OS default will be used.
+  config :receive_buffer_bytes, :validate => :number, :default => 32_768 # (32KB) Kafka default
+  # The base amount of time to wait before attempting to reconnect to a given host.
   # This avoids repeatedly connecting to a host in a tight loop.
-  # This backoff applies to all requests sent by the consumer to the broker.
-  config :reconnect_backoff_ms, :validate => :string
-  # The configuration controls the maximum amount of time the client will wait
-  # for the response of a request. If the response is not received before the timeout
-  # elapses the client will resend the request if necessary or fail the request if
-  # retries are exhausted.
-  config :request_timeout_ms, :validate => :string
+  # This backoff applies to all connection attempts by the client to a broker.
+  config :reconnect_backoff_ms, :validate => :number, :default => 50 # Kafka default
+  # The configuration controls the maximum amount of time the client will wait for the response of a request.
+  # If the response is not received before the timeout elapses the client will resend the request if necessary
+  # or fail the request if retries are exhausted.
+  config :request_timeout_ms, :validate => :number, :default => 40_000 # Kafka default
   # The amount of time to wait before attempting to retry a failed fetch request
   # to a given topic partition. This avoids repeated fetching-and-failing in a tight loop.
-  config :retry_backoff_ms, :validate => :string
-  # The size of the TCP send buffer (SO_SNDBUF) to use when sending data
-  config :send_buffer_bytes, :validate => :string
+  config :retry_backoff_ms, :validate => :number, :default => 100 # Kafka default
+  # The size of the TCP send buffer (SO_SNDBUF) to use when sending data.
+  # If the value is -1, the OS default will be used.
+  config :send_buffer_bytes, :validate => :number, :default => 131_072 # (128KB) Kafka default
   # The timeout after which, if the `poll_timeout_ms` is not invoked, the consumer is marked dead
   # and a rebalance operation is triggered for the group identified by `group_id`
-  config :session_timeout_ms, :validate => :string
+  config :session_timeout_ms, :validate => :number, :default => 10_000 # (10s) Kafka default
   # Java Class used to deserialize the record's value
   config :value_deserializer_class, :validate => :string, :default => "org.apache.kafka.common.serialization.StringDeserializer"
   # A list of topics to subscribe to, defaults to ["logstash"].
@@ -276,9 +286,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
             end
           end
           # Manual offset commit
-          if @enable_auto_commit == "false"
-            consumer.commitSync
-          end
+          consumer.commitSync if @enable_auto_commit.eql?(false)
         end
       rescue org.apache.kafka.common.errors.WakeupException => e
         raise e if !stop?
@@ -294,31 +302,33 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
       props = java.util.Properties.new
       kafka = org.apache.kafka.clients.consumer.ConsumerConfig
 
-      props.put(kafka::AUTO_COMMIT_INTERVAL_MS_CONFIG, auto_commit_interval_ms)
+      props.put(kafka::AUTO_COMMIT_INTERVAL_MS_CONFIG, auto_commit_interval_ms.to_s) unless auto_commit_interval_ms.nil?
       props.put(kafka::AUTO_OFFSET_RESET_CONFIG, auto_offset_reset) unless auto_offset_reset.nil?
       props.put(kafka::BOOTSTRAP_SERVERS_CONFIG, bootstrap_servers)
-      props.put(kafka::CHECK_CRCS_CONFIG, check_crcs) unless check_crcs.nil?
+      props.put(kafka::CHECK_CRCS_CONFIG, check_crcs.to_s) unless check_crcs.nil?
+      props.put(kafka::CLIENT_DNS_LOOKUP_CONFIG, client_dns_lookup)
       props.put(kafka::CLIENT_ID_CONFIG, client_id)
-      props.put(kafka::CONNECTIONS_MAX_IDLE_MS_CONFIG, connections_max_idle_ms) unless connections_max_idle_ms.nil?
-      props.put(kafka::ENABLE_AUTO_COMMIT_CONFIG, enable_auto_commit)
+      props.put(kafka::CONNECTIONS_MAX_IDLE_MS_CONFIG, connections_max_idle_ms.to_s) unless connections_max_idle_ms.nil?
+      props.put(kafka::ENABLE_AUTO_COMMIT_CONFIG, enable_auto_commit.to_s)
       props.put(kafka::EXCLUDE_INTERNAL_TOPICS_CONFIG, exclude_internal_topics) unless exclude_internal_topics.nil?
-      props.put(kafka::FETCH_MAX_BYTES_CONFIG, fetch_max_bytes) unless fetch_max_bytes.nil?
-      props.put(kafka::FETCH_MAX_WAIT_MS_CONFIG, fetch_max_wait_ms) unless fetch_max_wait_ms.nil?
-      props.put(kafka::FETCH_MIN_BYTES_CONFIG, fetch_min_bytes) unless fetch_min_bytes.nil?
+      props.put(kafka::FETCH_MAX_BYTES_CONFIG, fetch_max_bytes.to_s) unless fetch_max_bytes.nil?
+      props.put(kafka::FETCH_MAX_WAIT_MS_CONFIG, fetch_max_wait_ms.to_s) unless fetch_max_wait_ms.nil?
+      props.put(kafka::FETCH_MIN_BYTES_CONFIG, fetch_min_bytes.to_s) unless fetch_min_bytes.nil?
       props.put(kafka::GROUP_ID_CONFIG, group_id)
-      props.put(kafka::HEARTBEAT_INTERVAL_MS_CONFIG, heartbeat_interval_ms) unless heartbeat_interval_ms.nil?
+      props.put(kafka::HEARTBEAT_INTERVAL_MS_CONFIG, heartbeat_interval_ms.to_s) unless heartbeat_interval_ms.nil?
+      props.put(kafka::ISOLATION_LEVEL_CONFIG, isolation_level)
       props.put(kafka::KEY_DESERIALIZER_CLASS_CONFIG, key_deserializer_class)
-      props.put(kafka::MAX_PARTITION_FETCH_BYTES_CONFIG, max_partition_fetch_bytes) unless max_partition_fetch_bytes.nil?
-      props.put(kafka::MAX_POLL_RECORDS_CONFIG, max_poll_records) unless max_poll_records.nil?
-      props.put(kafka::MAX_POLL_INTERVAL_MS_CONFIG, max_poll_interval_ms) unless max_poll_interval_ms.nil?
-      props.put(kafka::METADATA_MAX_AGE_CONFIG, metadata_max_age_ms) unless metadata_max_age_ms.nil?
+      props.put(kafka::MAX_PARTITION_FETCH_BYTES_CONFIG, max_partition_fetch_bytes.to_s) unless max_partition_fetch_bytes.nil?
+      props.put(kafka::MAX_POLL_RECORDS_CONFIG, max_poll_records.to_s) unless max_poll_records.nil?
+      props.put(kafka::MAX_POLL_INTERVAL_MS_CONFIG, max_poll_interval_ms.to_s) unless max_poll_interval_ms.nil?
+      props.put(kafka::METADATA_MAX_AGE_CONFIG, metadata_max_age_ms.to_s) unless metadata_max_age_ms.nil?
       props.put(kafka::PARTITION_ASSIGNMENT_STRATEGY_CONFIG, partition_assignment_strategy_class) unless partition_assignment_strategy.nil?
-      props.put(kafka::RECEIVE_BUFFER_CONFIG, receive_buffer_bytes) unless receive_buffer_bytes.nil?
-      props.put(kafka::RECONNECT_BACKOFF_MS_CONFIG, reconnect_backoff_ms) unless reconnect_backoff_ms.nil?
-      props.put(kafka::REQUEST_TIMEOUT_MS_CONFIG, request_timeout_ms) unless request_timeout_ms.nil?
-      props.put(kafka::RETRY_BACKOFF_MS_CONFIG, retry_backoff_ms) unless retry_backoff_ms.nil?
-      props.put(kafka::SEND_BUFFER_CONFIG, send_buffer_bytes) unless send_buffer_bytes.nil?
-      props.put(kafka::SESSION_TIMEOUT_MS_CONFIG, session_timeout_ms) unless session_timeout_ms.nil?
+      props.put(kafka::RECEIVE_BUFFER_CONFIG, receive_buffer_bytes.to_s) unless receive_buffer_bytes.nil?
+      props.put(kafka::RECONNECT_BACKOFF_MS_CONFIG, reconnect_backoff_ms.to_s) unless reconnect_backoff_ms.nil?
+      props.put(kafka::REQUEST_TIMEOUT_MS_CONFIG, request_timeout_ms.to_s) unless request_timeout_ms.nil?
+      props.put(kafka::RETRY_BACKOFF_MS_CONFIG, retry_backoff_ms.to_s) unless retry_backoff_ms.nil?
+      props.put(kafka::SEND_BUFFER_CONFIG, send_buffer_bytes.to_s) unless send_buffer_bytes.nil?
+      props.put(kafka::SESSION_TIMEOUT_MS_CONFIG, session_timeout_ms.to_s) unless session_timeout_ms.nil?
       props.put(kafka::VALUE_DESERIALIZER_CLASS_CONFIG, value_deserializer_class)
       props.put(kafka::CLIENT_RACK_CONFIG, client_rack) unless client_rack.nil? 
 
@@ -374,15 +384,15 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   end
 
   def set_sasl_config(props)
-    java.lang.System.setProperty("java.security.auth.login.config",jaas_path) unless jaas_path.nil?
-    java.lang.System.setProperty("java.security.krb5.conf",kerberos_config) unless kerberos_config.nil?
+    java.lang.System.setProperty("java.security.auth.login.config", jaas_path) unless jaas_path.nil?
+    java.lang.System.setProperty("java.security.krb5.conf", kerberos_config) unless kerberos_config.nil?
 
-    props.put("sasl.mechanism",sasl_mechanism)
+    props.put("sasl.mechanism", sasl_mechanism)
     if sasl_mechanism == "GSSAPI" && sasl_kerberos_service_name.nil?
       raise LogStash::ConfigurationError, "sasl_kerberos_service_name must be specified when SASL mechanism is GSSAPI"
     end
 
-    props.put("sasl.kerberos.service.name",sasl_kerberos_service_name) unless sasl_kerberos_service_name.nil?
+    props.put("sasl.kerberos.service.name", sasl_kerberos_service_name) unless sasl_kerberos_service_name.nil?
     props.put("sasl.jaas.config", sasl_jaas_config) unless sasl_jaas_config.nil?
   end
 end #class LogStash::Inputs::Kafka

data/lib/logstash/outputs/kafka.rb

@@ -67,7 +67,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   # The producer will attempt to batch records together into fewer requests whenever multiple
   # records are being sent to the same partition. This helps performance on both the client
   # and the server. This configuration controls the default batch size in bytes.
-  config :batch_size, :validate => :number, :default => 16384
+  config :batch_size, :validate => :number, :default => 16_384 # Kafka default
   # This is for bootstrapping and the producer will only use it for getting metadata (topics,
   # partitions and replicas). The socket connections for sending the actual data will be
   # established based on the broker information returned in the metadata. The format is
@@ -75,10 +75,15 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   # subset of brokers.
   config :bootstrap_servers, :validate => :string, :default => 'localhost:9092'
   # The total bytes of memory the producer can use to buffer records waiting to be sent to the server.
-  config :buffer_memory, :validate => :number, :default => 33554432
+  config :buffer_memory, :validate => :number, :default => 33_554_432 # (32M) Kafka default
   # The compression type for all data generated by the producer.
   # The default is none (i.e. no compression). Valid values are none, gzip, or snappy.
   config :compression_type, :validate => ["none", "gzip", "snappy", "lz4"], :default => "none"
+  # How DNS lookups should be done. If set to `use_all_dns_ips`, when the lookup returns multiple 
+  # IP addresses for a hostname, they will all be attempted to connect to before failing the 
+  # connection. If the value is `resolve_canonical_bootstrap_servers_only` each entry will be 
+  # resolved and expanded into a list of canonical names.
+  config :client_dns_lookup, :validate => ["default", "use_all_dns_ips", "resolve_canonical_bootstrap_servers_only"], :default => "default"
   # The id string to pass to the server when making requests.
   # The purpose of this is to be able to track the source of requests beyond just
   # ip/port by allowing a logical application name to be included with the request
@@ -92,26 +97,26 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   # This setting accomplishes this by adding a small amount of artificial delay—that is,
   # rather than immediately sending out a record the producer will wait for up to the given delay
   # to allow other records to be sent so that the sends can be batched together.
-  config :linger_ms, :validate => :number, :default => 0
+  config :linger_ms, :validate => :number, :default => 0 # Kafka default
   # The maximum size of a request
-  config :max_request_size, :validate => :number, :default => 1048576
+  config :max_request_size, :validate => :number, :default => 1_048_576 # (1MB) Kafka default
   # The key for the message
   config :message_key, :validate => :string
   # the timeout setting for initial metadata request to fetch topic metadata.
-  config :metadata_fetch_timeout_ms, :validate => :number, :default => 60000
+  config :metadata_fetch_timeout_ms, :validate => :number, :default => 60_000
   # the max time in milliseconds before a metadata refresh is forced.
-  config :metadata_max_age_ms, :validate => :number, :default => 300000
+  config :metadata_max_age_ms, :validate => :number, :default => 300_000 # (5m) Kafka default
   # Partitioner to use - can be `default`, `uniform_sticky`, `round_robin` or a fully qualified class name of a custom partitioner.
   config :partitioner, :validate => :string
   # The size of the TCP receive buffer to use when reading data
-  config :receive_buffer_bytes, :validate => :number, :default => 32768
+  config :receive_buffer_bytes, :validate => :number, :default => 32_768 # (32KB) Kafka default
   # The amount of time to wait before attempting to reconnect to a given host when a connection fails.
-  config :reconnect_backoff_ms, :validate => :number, :default => 10
+  config :reconnect_backoff_ms, :validate => :number, :default => 50 # Kafka default
   # The configuration controls the maximum amount of time the client will wait
   # for the response of a request. If the response is not received before the timeout
   # elapses the client will resend the request if necessary or fail the request if
   # retries are exhausted.
-  config :request_timeout_ms, :validate => :string
+  config :request_timeout_ms, :validate => :number, :default => 40_000 # (40s) Kafka default
   # The default retry behavior is to retry until successful. To prevent data loss,
   # the use of this setting is discouraged.
   #
@@ -122,9 +127,9 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   # A value less than zero is a configuration error.
   config :retries, :validate => :number
   # The amount of time to wait before attempting to retry a failed produce request to a given topic partition.
-  config :retry_backoff_ms, :validate => :number, :default => 100
+  config :retry_backoff_ms, :validate => :number, :default => 100 # Kafka default
   # The size of the TCP send buffer to use when sending data.
-  config :send_buffer_bytes, :validate => :number, :default => 131072
+  config :send_buffer_bytes, :validate => :number, :default => 131_072 # (128KB) Kafka default
   # The truststore type.
   config :ssl_truststore_type, :validate => :string
   # The JKS truststore path to validate the Kafka broker's certificate.
@@ -231,7 +236,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
     remaining = @retries
 
     while batch.any?
-      if !remaining.nil?
+      unless remaining.nil?
         if remaining < 0
           # TODO(sissel): Offer to DLQ? Then again, if it's a transient fault,
           # DLQing would make things worse (you dlq data that would be successful
@@ -250,27 +255,39 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
         begin
           # send() can throw an exception even before the future is created.
           @producer.send(record)
-        rescue org.apache.kafka.common.errors.TimeoutException => e
+        rescue org.apache.kafka.common.errors.InterruptException,
+               org.apache.kafka.common.errors.RetriableException => e
+          logger.info("producer send failed, will retry sending", :exception => e.class, :message => e.message)
           failures << record
           nil
-        rescue org.apache.kafka.common.errors.InterruptException => e
-          failures << record
-          nil
-        rescue org.apache.kafka.common.errors.SerializationException => e
-          # TODO(sissel): Retrying will fail because the data itself has a problem serializing.
-          # TODO(sissel): Let's add DLQ here.
-          failures << record
+        rescue org.apache.kafka.common.KafkaException => e
+          # This error is not retriable, drop event
+          # TODO: add DLQ support
+          logger.warn("producer send failed, dropping record",:exception => e.class, :message => e.message,
+                      :record_value => record.value)
           nil
         end
-      end.compact
+      end
 
       futures.each_with_index do |future, i|
-        begin
-          result = future.get()
-        rescue => e
-          # TODO(sissel): Add metric to count failures, possibly by exception type.
-          logger.warn("producer send failed", :exception => e.class, :message => e.message)
-          failures << batch[i]
+        # We cannot skip nils using `futures.compact` because then our index `i` will not align with `batch`
+        unless future.nil?
+          begin
+            future.get
+          rescue java.util.concurrent.ExecutionException => e
+            # TODO(sissel): Add metric to count failures, possibly by exception type.
+            if e.get_cause.is_a? org.apache.kafka.common.errors.RetriableException or
+               e.get_cause.is_a? org.apache.kafka.common.errors.InterruptException
+              logger.info("producer send failed, will retry sending", :exception => e.cause.class,
+                          :message => e.cause.message)
+              failures << batch[i]
+            elsif e.get_cause.is_a? org.apache.kafka.common.KafkaException
+              # This error is not retriable, drop event
+              # TODO: add DLQ support
+              logger.warn("producer send failed, dropping record", :exception => e.cause.class,
+                          :message => e.cause.message, :record_value => batch[i].value)
+            end
+          end
         end
       end
 
@@ -318,18 +335,19 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
       props.put(kafka::BOOTSTRAP_SERVERS_CONFIG, bootstrap_servers)
       props.put(kafka::BUFFER_MEMORY_CONFIG, buffer_memory.to_s)
       props.put(kafka::COMPRESSION_TYPE_CONFIG, compression_type)
+      props.put(kafka::CLIENT_DNS_LOOKUP_CONFIG, client_dns_lookup)
       props.put(kafka::CLIENT_ID_CONFIG, client_id) unless client_id.nil?
       props.put(kafka::KEY_SERIALIZER_CLASS_CONFIG, key_serializer)
       props.put(kafka::LINGER_MS_CONFIG, linger_ms.to_s)
       props.put(kafka::MAX_REQUEST_SIZE_CONFIG, max_request_size.to_s)
-      props.put(kafka::METADATA_MAX_AGE_CONFIG, metadata_max_age_ms) unless metadata_max_age_ms.nil?
+      props.put(kafka::METADATA_MAX_AGE_CONFIG, metadata_max_age_ms.to_s) unless metadata_max_age_ms.nil?
       unless partitioner.nil?
         props.put(kafka::PARTITIONER_CLASS_CONFIG, partitioner = partitioner_class)
         logger.debug('producer configured using partitioner', :partitioner_class => partitioner)
       end
       props.put(kafka::RECEIVE_BUFFER_CONFIG, receive_buffer_bytes.to_s) unless receive_buffer_bytes.nil?
-      props.put(kafka::RECONNECT_BACKOFF_MS_CONFIG, reconnect_backoff_ms) unless reconnect_backoff_ms.nil?
-      props.put(kafka::REQUEST_TIMEOUT_MS_CONFIG, request_timeout_ms) unless request_timeout_ms.nil?
+      props.put(kafka::RECONNECT_BACKOFF_MS_CONFIG, reconnect_backoff_ms.to_s) unless reconnect_backoff_ms.nil?
+      props.put(kafka::REQUEST_TIMEOUT_MS_CONFIG, request_timeout_ms.to_s) unless request_timeout_ms.nil?
       props.put(kafka::RETRIES_CONFIG, retries.to_s) unless retries.nil?
       props.put(kafka::RETRY_BACKOFF_MS_CONFIG, retry_backoff_ms.to_s) 
       props.put(kafka::SEND_BUFFER_CONFIG, send_buffer_bytes.to_s)
@@ -346,7 +364,6 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
         set_sasl_config(props)
       end
 
-
       org.apache.kafka.clients.producer.KafkaProducer.new(props)
     rescue => e
       logger.error("Unable to create Kafka producer from given configuration",