logstash-integration-kafka 10.0.0-java

data/docs/output-kafka.asciidoc

@@ -0,0 +1,441 @@
+:plugin: kafka
+:type: output
+:default_codec: plain
+
+///////////////////////////////////////////
+START - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+:version: %VERSION%
+:release_date: %RELEASE_DATE%
+:changelog_url: %CHANGELOG_URL%
+:include_path: ../../../../logstash/docs/include
+///////////////////////////////////////////
+END - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+
+[id="plugins-{type}s-{plugin}"]
+
+=== Kafka output plugin
+
+include::{include_path}/plugin_header.asciidoc[]
+
+==== Description
+
+Write events to a Kafka topic. 
+
+This plugin uses Kafka Client 2.1.0. For broker compatibility, see the official https://cwiki.apache.org/confluence/display/KAFKA/Compatibility+Matrix[Kafka compatibility reference]. If the linked compatibility wiki is not up-to-date, please contact Kafka support/community to confirm compatibility.
+
+If you require features not yet available in this plugin (including client version upgrades), please file an issue with details about what you need.
+
+This output supports connecting to Kafka over:
+
+* SSL (requires plugin version 3.0.0 or later)
+* Kerberos SASL (requires plugin version 5.1.0 or later)
+
+By default security is disabled but can be turned on as needed.
+
+The only required configuration is the topic_id. 
+
+The default codec is plain. Logstash will encode your events with not only the message field but also with a timestamp and hostname.
+
+If you want the full content of your events to be sent as json, you should set the codec in the output configuration like this:
+[source,ruby]
+    output {
+      kafka {
+        codec => json
+        topic_id => "mytopic"
+      }
+    }
+    
+For more information see http://kafka.apache.org/documentation.html#theproducer
+
+Kafka producer configuration: http://kafka.apache.org/documentation.html#newproducerconfigs
+
+[id="plugins-{type}s-{plugin}-options"]
+==== Kafka Output Configuration Options
+
+This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-acks>> |<<string,string>>, one of `["0", "1", "all"]`|No
+| <<plugins-{type}s-{plugin}-batch_size>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-bootstrap_servers>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-buffer_memory>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-client_id>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-compression_type>> |<<string,string>>, one of `["none", "gzip", "snappy", "lz4"]`|No
+| <<plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-key_serializer>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-linger_ms>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-max_request_size>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-message_key>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-metadata_fetch_timeout_ms>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-metadata_max_age_ms>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-receive_buffer_bytes>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-reconnect_backoff_ms>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-request_timeout_ms>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-retries>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-retry_backoff_ms>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-sasl_jaas_config>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-sasl_mechanism>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-security_protocol>> |<<string,string>>, one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No
+| <<plugins-{type}s-{plugin}-send_buffer_bytes>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-ssl_key_password>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-ssl_keystore_password>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-ssl_keystore_type>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-ssl_truststore_password>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-ssl_truststore_type>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-topic_id>> |<<string,string>>|Yes
+| <<plugins-{type}s-{plugin}-value_serializer>> |<<string,string>>|No
+|=======================================================================
+
+Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
+output plugins.
+
+&nbsp;
+
+[id="plugins-{type}s-{plugin}-acks"]
+===== `acks` 
+
+  * Value can be any of: `0`, `1`, `all`
+  * Default value is `"1"`
+
+The number of acknowledgments the producer requires the leader to have received
+before considering a request complete.
+
+acks=0,   the producer will not wait for any acknowledgment from the server at all.
+acks=1,   This will mean the leader will write the record to its local log but
+          will respond without awaiting full acknowledgement from all followers.
+acks=all, This means the leader will wait for the full set of in-sync replicas to acknowledge the record.
+
+[id="plugins-{type}s-{plugin}-batch_size"]
+===== `batch_size` 
+
+  * Value type is <<number,number>>
+  * Default value is `16384`
+
+The producer will attempt to batch records together into fewer requests whenever multiple
+records are being sent to the same partition. This helps performance on both the client
+and the server. This configuration controls the default batch size in bytes.
+
+[id="plugins-{type}s-{plugin}-bootstrap_servers"]
+===== `bootstrap_servers` 
+
+  * Value type is <<string,string>>
+  * Default value is `"localhost:9092"`
+
+This is for bootstrapping and the producer will only use it for getting metadata (topics,
+partitions and replicas). The socket connections for sending the actual data will be
+established based on the broker information returned in the metadata. The format is
+`host1:port1,host2:port2`, and the list can be a subset of brokers or a VIP pointing to a
+subset of brokers.
+
+[id="plugins-{type}s-{plugin}-buffer_memory"]
+===== `buffer_memory` 
+
+  * Value type is <<number,number>>
+  * Default value is `33554432`
+
+The total bytes of memory the producer can use to buffer records waiting to be sent to the server.
+
+[id="plugins-{type}s-{plugin}-client_id"]
+===== `client_id` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The id string to pass to the server when making requests.
+The purpose of this is to be able to track the source of requests beyond just
+ip/port by allowing a logical application name to be included with the request
+
+[id="plugins-{type}s-{plugin}-compression_type"]
+===== `compression_type` 
+
+  * Value can be any of: `none`, `gzip`, `snappy`, `lz4`
+  * Default value is `"none"`
+
+The compression type for all data generated by the producer.
+The default is none (i.e. no compression). Valid values are none, gzip, or snappy.
+
+[id="plugins-{type}s-{plugin}-jaas_path"]
+===== `jaas_path` 
+
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
+
+The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization 
+services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client:
+[source,java]
+----------------------------------
+KafkaClient {
+  com.sun.security.auth.module.Krb5LoginModule required
+  useTicketCache=true
+  renewTicket=true
+  serviceName="kafka";
+  };
+----------------------------------
+
+Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these 
+to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same 
+`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on 
+different JVM instances.
+
+[id="plugins-{type}s-{plugin}-kerberos_config"]
+===== `kerberos_config` 
+
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
+
+Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html
+
+[id="plugins-{type}s-{plugin}-key_serializer"]
+===== `key_serializer` 
+
+  * Value type is <<string,string>>
+  * Default value is `"org.apache.kafka.common.serialization.StringSerializer"`
+
+Serializer class for the key of the message
+
+[id="plugins-{type}s-{plugin}-linger_ms"]
+===== `linger_ms` 
+
+  * Value type is <<number,number>>
+  * Default value is `0`
+
+The producer groups together any records that arrive in between request
+transmissions into a single batched request. Normally this occurs only under
+load when records arrive faster than they can be sent out. However in some circumstances
+the client may want to reduce the number of requests even under moderate load.
+This setting accomplishes this by adding a small amount of artificial delay—that is,
+rather than immediately sending out a record the producer will wait for up to the given delay
+to allow other records to be sent so that the sends can be batched together.
+
+[id="plugins-{type}s-{plugin}-max_request_size"]
+===== `max_request_size` 
+
+  * Value type is <<number,number>>
+  * Default value is `1048576`
+
+The maximum size of a request
+
+[id="plugins-{type}s-{plugin}-message_key"]
+===== `message_key` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The key for the message
+
+[id="plugins-{type}s-{plugin}-metadata_fetch_timeout_ms"]
+===== `metadata_fetch_timeout_ms` 
+
+  * Value type is <<number,number>>
+  * Default value is `60000`
+
+the timeout setting for initial metadata request to fetch topic metadata.
+
+[id="plugins-{type}s-{plugin}-metadata_max_age_ms"]
+===== `metadata_max_age_ms` 
+
+  * Value type is <<number,number>>
+  * Default value is `300000`
+
+the max time in milliseconds before a metadata refresh is forced.
+
+[id="plugins-{type}s-{plugin}-receive_buffer_bytes"]
+===== `receive_buffer_bytes` 
+
+  * Value type is <<number,number>>
+  * Default value is `32768`
+
+The size of the TCP receive buffer to use when reading data
+
+[id="plugins-{type}s-{plugin}-reconnect_backoff_ms"]
+===== `reconnect_backoff_ms` 
+
+  * Value type is <<number,number>>
+  * Default value is `10`
+
+The amount of time to wait before attempting to reconnect to a given host when a connection fails.
+
+[id="plugins-{type}s-{plugin}-request_timeout_ms"]
+===== `request_timeout_ms` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The configuration controls the maximum amount of time the client will wait
+for the response of a request. If the response is not received before the timeout
+elapses the client will resend the request if necessary or fail the request if
+retries are exhausted.
+
+[id="plugins-{type}s-{plugin}-retries"]
+===== `retries` 
+
+  * Value type is <<number,number>>
+  * There is no default value for this setting.
+
+The default retry behavior is to retry until successful. To prevent data loss,
+the use of this setting is discouraged.
+
+If you choose to set `retries`, a value greater than zero will cause the
+client to only retry a fixed number of times. This will result in data loss
+if a transport fault exists for longer than your retry count (network outage,
+Kafka down, etc).
+
+A value less than zero is a configuration error.
+
+[id="plugins-{type}s-{plugin}-retry_backoff_ms"]
+===== `retry_backoff_ms` 
+
+  * Value type is <<number,number>>
+  * Default value is `100`
+
+The amount of time to wait before attempting to retry a failed produce request to a given topic partition.
+
+[id="plugins-{type}s-{plugin}-sasl_jaas_config"]
+===== `sasl_jaas_config` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+JAAS configuration setting local to this plugin instance, as opposed to settings using config file configured using `jaas_path`, which are shared across the JVM. This allows each plugin instance to have its own configuration. 
+
+If both `sasl_jaas_config` and `jaas_path` configurations are set, the setting here takes precedence.
+
+Example (setting for Azure Event Hub):
+[source,ruby]
+    output {
+      kafka {
+        sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='auser'  password='apassword';"
+      }
+    }
+
+[id="plugins-{type}s-{plugin}-sasl_kerberos_service_name"]
+===== `sasl_kerberos_service_name` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The Kerberos principal name that Kafka broker runs as. 
+This can be defined either in Kafka's JAAS config or in Kafka's config.
+
+[id="plugins-{type}s-{plugin}-sasl_mechanism"]
+===== `sasl_mechanism` 
+
+  * Value type is <<string,string>>
+  * Default value is `"GSSAPI"`
+
+http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. 
+This may be any mechanism for which a security provider is available.
+GSSAPI is the default mechanism.
+
+[id="plugins-{type}s-{plugin}-security_protocol"]
+===== `security_protocol` 
+
+  * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL`
+  * Default value is `"PLAINTEXT"`
+
+Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL
+
+[id="plugins-{type}s-{plugin}-send_buffer_bytes"]
+===== `send_buffer_bytes` 
+
+  * Value type is <<number,number>>
+  * Default value is `131072`
+
+The size of the TCP send buffer to use when sending data.
+
+[id="plugins-{type}s-{plugin}-ssl_endpoint_identification_algorithm"]
+===== `ssl_endpoint_identification_algorithm`
+
+  * Value type is <<string,string>>
+  * Default value is `"https"`
+
+The endpoint identification algorithm, defaults to `"https"`. Set to empty string `""` to disable
+
+[id="plugins-{type}s-{plugin}-ssl_key_password"]
+===== `ssl_key_password` 
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+The password of the private key in the key store file.
+
+[id="plugins-{type}s-{plugin}-ssl_keystore_location"]
+===== `ssl_keystore_location` 
+
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
+
+If client authentication is required, this setting stores the keystore path.
+
+[id="plugins-{type}s-{plugin}-ssl_keystore_password"]
+===== `ssl_keystore_password` 
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+If client authentication is required, this setting stores the keystore password
+
+[id="plugins-{type}s-{plugin}-ssl_keystore_type"]
+===== `ssl_keystore_type` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The keystore type.
+
+[id="plugins-{type}s-{plugin}-ssl_truststore_location"]
+===== `ssl_truststore_location` 
+
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
+
+The JKS truststore path to validate the Kafka broker's certificate.
+
+[id="plugins-{type}s-{plugin}-ssl_truststore_password"]
+===== `ssl_truststore_password` 
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+The truststore password
+
+[id="plugins-{type}s-{plugin}-ssl_truststore_type"]
+===== `ssl_truststore_type` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The truststore type.
+
+[id="plugins-{type}s-{plugin}-topic_id"]
+===== `topic_id` 
+
+  * This is a required setting.
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The topic to produce messages to
+
+[id="plugins-{type}s-{plugin}-value_serializer"]
+===== `value_serializer` 
+
+  * Value type is <<string,string>>
+  * Default value is `"org.apache.kafka.common.serialization.StringSerializer"`
+
+Serializer class for the value of the message
+
+
+
+[id="plugins-{type}s-{plugin}-common-options"]
+include::{include_path}/{type}.asciidoc[]
+
+:default_codec!:

data/lib/logstash-integration-kafka_jars.rb

@@ -0,0 +1,8 @@
+# AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
+
+require 'jar_dependencies'
+require_jar('org.apache.kafka', 'kafka-clients', '2.3.0')
+require_jar('com.github.luben', 'zstd-jni', '1.4.2-1')
+require_jar('org.slf4j', 'slf4j-api', '1.7.26')
+require_jar('org.lz4', 'lz4-java', '1.6.0')
+require_jar('org.xerial.snappy', 'snappy-java', '1.1.7.3')

data/lib/logstash/inputs/kafka.rb

@@ -0,0 +1,362 @@
+require 'logstash/namespace'
+require 'logstash/inputs/base'
+require 'stud/interval'
+require 'java'
+require 'logstash-integration-kafka_jars.rb'
+
+# This input will read events from a Kafka topic. It uses the 0.10 version of
+# the consumer API provided by Kafka to read messages from the broker.
+#
+# Here's a compatibility matrix that shows the Kafka client versions that are compatible with each combination
+# of Logstash and the Kafka input plugin: 
+# 
+# [options="header"]
+# |==========================================================
+# |Kafka Client Version |Logstash Version |Plugin Version |Why?
+# |0.8       |2.0.0 - 2.x.x   |<3.0.0 |Legacy, 0.8 is still popular 
+# |0.9       |2.0.0 - 2.3.x   | 3.x.x |Works with the old Ruby Event API (`event['product']['price'] = 10`)  
+# |0.9       |2.4.x - 5.x.x   | 4.x.x |Works with the new getter/setter APIs (`event.set('[product][price]', 10)`)
+# |0.10.0.x  |2.4.x - 5.x.x   | 5.x.x |Not compatible with the <= 0.9 broker
+# |0.10.1.x  |2.4.x - 5.x.x   | 6.x.x |
+# |==========================================================
+# 
+# NOTE: We recommended that you use matching Kafka client and broker versions. During upgrades, you should
+# upgrade brokers before clients because brokers target backwards compatibility. For example, the 0.9 broker
+# is compatible with both the 0.8 consumer and 0.9 consumer APIs, but not the other way around.
+#
+# This input supports connecting to Kafka over:
+#
+# * SSL (requires plugin version 3.0.0 or later)
+# * Kerberos SASL (requires plugin version 5.1.0 or later) 
+#
+# By default security is disabled but can be turned on as needed.
+#
+# The Logstash Kafka consumer handles group management and uses the default offset management
+# strategy using Kafka topics.
+#
+# Logstash instances by default form a single logical group to subscribe to Kafka topics
+# Each Logstash Kafka consumer can run multiple threads to increase read throughput. Alternatively, 
+# you could run multiple Logstash instances with the same `group_id` to spread the load across
+# physical machines. Messages in a topic will be distributed to all Logstash instances with
+# the same `group_id`.
+#
+# Ideally you should have as many threads as the number of partitions for a perfect balance --
+# more threads than partitions means that some threads will be idle
+#
+# For more information see http://kafka.apache.org/documentation.html#theconsumer
+#
+# Kafka consumer configuration: http://kafka.apache.org/documentation.html#consumerconfigs
+#
+class LogStash::Inputs::Kafka < LogStash::Inputs::Base
+  config_name 'kafka'
+
+  default :codec, 'plain'
+
+  # The frequency in milliseconds that the consumer offsets are committed to Kafka.
+  config :auto_commit_interval_ms, :validate => :string, :default => "5000"
+  # What to do when there is no initial offset in Kafka or if an offset is out of range:
+  #
+  # * earliest: automatically reset the offset to the earliest offset
+  # * latest: automatically reset the offset to the latest offset
+  # * none: throw exception to the consumer if no previous offset is found for the consumer's group
+  # * anything else: throw exception to the consumer.
+  config :auto_offset_reset, :validate => :string
+  # A list of URLs of Kafka instances to use for establishing the initial connection to the cluster.
+  # This list should be in the form of `host1:port1,host2:port2` These urls are just used
+  # for the initial connection to discover the full cluster membership (which may change dynamically)
+  # so this list need not contain the full set of servers (you may want more than one, though, in
+  # case a server is down).
+  config :bootstrap_servers, :validate => :string, :default => "localhost:9092"
+  # Automatically check the CRC32 of the records consumed. This ensures no on-the-wire or on-disk
+  # corruption to the messages occurred. This check adds some overhead, so it may be
+  # disabled in cases seeking extreme performance.
+  config :check_crcs, :validate => :string
+  # The id string to pass to the server when making requests. The purpose of this
+  # is to be able to track the source of requests beyond just ip/port by allowing
+  # a logical application name to be included.
+  config :client_id, :validate => :string, :default => "logstash"
+  # Close idle connections after the number of milliseconds specified by this config.
+  config :connections_max_idle_ms, :validate => :string
+  # Ideally you should have as many threads as the number of partitions for a perfect
+  # balance — more threads than partitions means that some threads will be idle
+  config :consumer_threads, :validate => :number, :default => 1
+  # If true, periodically commit to Kafka the offsets of messages already returned by the consumer. 
+  # This committed offset will be used when the process fails as the position from
+  # which the consumption will begin.
+  config :enable_auto_commit, :validate => :string, :default => "true"
+  # Whether records from internal topics (such as offsets) should be exposed to the consumer.
+  # If set to true the only way to receive records from an internal topic is subscribing to it.
+  config :exclude_internal_topics, :validate => :string
+  # The maximum amount of data the server should return for a fetch request. This is not an 
+  # absolute maximum, if the first message in the first non-empty partition of the fetch is larger 
+  # than this value, the message will still be returned to ensure that the consumer can make progress.
+  config :fetch_max_bytes, :validate => :string
+  # The maximum amount of time the server will block before answering the fetch request if
+  # there isn't sufficient data to immediately satisfy `fetch_min_bytes`. This
+  # should be less than or equal to the timeout used in `poll_timeout_ms`
+  config :fetch_max_wait_ms, :validate => :string
+  # The minimum amount of data the server should return for a fetch request. If insufficient
+  # data is available the request will wait for that much data to accumulate
+  # before answering the request.
+  config :fetch_min_bytes, :validate => :string
+  # The identifier of the group this consumer belongs to. Consumer group is a single logical subscriber
+  # that happens to be made up of multiple processors. Messages in a topic will be distributed to all
+  # Logstash instances with the same `group_id`
+  config :group_id, :validate => :string, :default => "logstash"
+  # The expected time between heartbeats to the consumer coordinator. Heartbeats are used to ensure 
+  # that the consumer's session stays active and to facilitate rebalancing when new
+  # consumers join or leave the group. The value must be set lower than
+  # `session.timeout.ms`, but typically should be set no higher than 1/3 of that value.
+  # It can be adjusted even lower to control the expected time for normal rebalances.
+  config :heartbeat_interval_ms, :validate => :string
+  # Java Class used to deserialize the record's key
+  config :key_deserializer_class, :validate => :string, :default => "org.apache.kafka.common.serialization.StringDeserializer"
+  # The maximum delay between invocations of poll() when using consumer group management. This places 
+  # an upper bound on the amount of time that the consumer can be idle before fetching more records. 
+  # If poll() is not called before expiration of this timeout, then the consumer is considered failed and 
+  # the group will rebalance in order to reassign the partitions to another member.
+  # The value of the configuration `request_timeout_ms` must always be larger than max_poll_interval_ms
+  config :max_poll_interval_ms, :validate => :string
+  # The maximum amount of data per-partition the server will return. The maximum total memory used for a
+  # request will be <code>#partitions * max.partition.fetch.bytes</code>. This size must be at least
+  # as large as the maximum message size the server allows or else it is possible for the producer to
+  # send messages larger than the consumer can fetch. If that happens, the consumer can get stuck trying
+  # to fetch a large message on a certain partition.
+  config :max_partition_fetch_bytes, :validate => :string
+  # The maximum number of records returned in a single call to poll().
+  config :max_poll_records, :validate => :string
+  # The period of time in milliseconds after which we force a refresh of metadata even if
+  # we haven't seen any partition leadership changes to proactively discover any new brokers or partitions
+  config :metadata_max_age_ms, :validate => :string
+  # The class name of the partition assignment strategy that the client will use to distribute
+  # partition ownership amongst consumer instances
+  config :partition_assignment_strategy, :validate => :string
+  # The size of the TCP receive buffer (SO_RCVBUF) to use when reading data.
+  config :receive_buffer_bytes, :validate => :string
+  # The amount of time to wait before attempting to reconnect to a given host.
+  # This avoids repeatedly connecting to a host in a tight loop.
+  # This backoff applies to all requests sent by the consumer to the broker.
+  config :reconnect_backoff_ms, :validate => :string
+  # The configuration controls the maximum amount of time the client will wait
+  # for the response of a request. If the response is not received before the timeout
+  # elapses the client will resend the request if necessary or fail the request if
+  # retries are exhausted.
+  config :request_timeout_ms, :validate => :string
+  # The amount of time to wait before attempting to retry a failed fetch request
+  # to a given topic partition. This avoids repeated fetching-and-failing in a tight loop.
+  config :retry_backoff_ms, :validate => :string
+  # The size of the TCP send buffer (SO_SNDBUF) to use when sending data
+  config :send_buffer_bytes, :validate => :string
+  # The timeout after which, if the `poll_timeout_ms` is not invoked, the consumer is marked dead
+  # and a rebalance operation is triggered for the group identified by `group_id`
+  config :session_timeout_ms, :validate => :string
+  # Java Class used to deserialize the record's value
+  config :value_deserializer_class, :validate => :string, :default => "org.apache.kafka.common.serialization.StringDeserializer"
+  # A list of topics to subscribe to, defaults to ["logstash"].
+  config :topics, :validate => :array, :default => ["logstash"]
+  # A topic regex pattern to subscribe to. 
+  # The topics configuration will be ignored when using this configuration.
+  config :topics_pattern, :validate => :string
+  # Time kafka consumer will wait to receive new messages from topics
+  config :poll_timeout_ms, :validate => :number, :default => 100
+  # The truststore type.
+  config :ssl_truststore_type, :validate => :string
+  # The JKS truststore path to validate the Kafka broker's certificate.
+  config :ssl_truststore_location, :validate => :path
+  # The truststore password
+  config :ssl_truststore_password, :validate => :password
+  # The keystore type.
+  config :ssl_keystore_type, :validate => :string
+  # If client authentication is required, this setting stores the keystore path.
+  config :ssl_keystore_location, :validate => :path
+  # If client authentication is required, this setting stores the keystore password
+  config :ssl_keystore_password, :validate => :password
+  # The password of the private key in the key store file.
+  config :ssl_key_password, :validate => :password
+  # Algorithm to use when verifying host. Set to "" to disable
+  config :ssl_endpoint_identification_algorithm, :validate => :string, :default => 'https'
+  # Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL
+  config :security_protocol, :validate => ["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"], :default => "PLAINTEXT"
+  # http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. 
+  # This may be any mechanism for which a security provider is available.
+  # GSSAPI is the default mechanism.
+  config :sasl_mechanism, :validate => :string, :default => "GSSAPI"
+  # The Kerberos principal name that Kafka broker runs as. 
+  # This can be defined either in Kafka's JAAS config or in Kafka's config.
+  config :sasl_kerberos_service_name, :validate => :string
+  # The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization 
+  # services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client:
+  # [source,java]
+  # ----------------------------------
+  # KafkaClient {
+  #   com.sun.security.auth.module.Krb5LoginModule required
+  #   useTicketCache=true
+  #   renewTicket=true
+  #   serviceName="kafka";
+  #   };
+  # ----------------------------------
+  #
+  # Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these  
+  # to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same 
+  # `jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on 
+  # different JVM instances.
+  config :jaas_path, :validate => :path
+  # JAAS configuration settings. This allows JAAS config to be a part of the plugin configuration and allows for different JAAS configuration per each plugin config.
+  config :sasl_jaas_config, :validate => :string
+  # Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html
+  config :kerberos_config, :validate => :path
+  # Option to add Kafka metadata like topic, message size to the event.
+  # This will add a field named `kafka` to the logstash event containing the following attributes:
+  #   `topic`: The topic this message is associated with
+  #   `consumer_group`: The consumer group used to read in this event
+  #   `partition`: The partition this message is associated with
+  #   `offset`: The offset from the partition this message is associated with
+  #   `key`: A ByteBuffer containing the message key
+  #   `timestamp`: The timestamp of this message
+  config :decorate_events, :validate => :boolean, :default => false
+
+
+  public
+  def register
+    @runner_threads = []
+  end # def register
+
+  public
+  def run(logstash_queue)
+    @runner_consumers = consumer_threads.times.map { |i| create_consumer("#{client_id}-#{i}") }
+    @runner_threads = @runner_consumers.map { |consumer| thread_runner(logstash_queue, consumer) }
+    @runner_threads.each { |t| t.join }
+  end # def run
+
+  public
+  def stop
+    # if we have consumers, wake them up to unblock our runner threads
+    @runner_consumers && @runner_consumers.each(&:wakeup)
+  end
+
+  public
+  def kafka_consumers
+    @runner_consumers
+  end
+
+  private
+  def thread_runner(logstash_queue, consumer)
+    Thread.new do
+      begin
+        unless @topics_pattern.nil?
+          nooplistener = org.apache.kafka.clients.consumer.internals.NoOpConsumerRebalanceListener.new
+          pattern = java.util.regex.Pattern.compile(@topics_pattern)
+          consumer.subscribe(pattern, nooplistener)
+        else
+          consumer.subscribe(topics);
+        end
+        codec_instance = @codec.clone
+        while !stop?
+          records = consumer.poll(poll_timeout_ms)
+          next unless records.count > 0
+          for record in records do
+            codec_instance.decode(record.value.to_s) do |event|
+              decorate(event)
+              if @decorate_events
+                event.set("[@metadata][kafka][topic]", record.topic)
+                event.set("[@metadata][kafka][consumer_group]", @group_id)
+                event.set("[@metadata][kafka][partition]", record.partition)
+                event.set("[@metadata][kafka][offset]", record.offset)
+                event.set("[@metadata][kafka][key]", record.key)
+                event.set("[@metadata][kafka][timestamp]", record.timestamp)
+              end
+              logstash_queue << event
+            end
+          end
+          # Manual offset commit
+          if @enable_auto_commit == "false"
+            consumer.commitSync
+          end
+        end
+      rescue org.apache.kafka.common.errors.WakeupException => e
+        raise e if !stop?
+      ensure
+        consumer.close
+      end
+    end
+  end
+
+  private
+  def create_consumer(client_id)
+    begin
+      props = java.util.Properties.new
+      kafka = org.apache.kafka.clients.consumer.ConsumerConfig
+
+      props.put(kafka::AUTO_COMMIT_INTERVAL_MS_CONFIG, auto_commit_interval_ms)
+      props.put(kafka::AUTO_OFFSET_RESET_CONFIG, auto_offset_reset) unless auto_offset_reset.nil?
+      props.put(kafka::BOOTSTRAP_SERVERS_CONFIG, bootstrap_servers)
+      props.put(kafka::CHECK_CRCS_CONFIG, check_crcs) unless check_crcs.nil?
+      props.put(kafka::CLIENT_ID_CONFIG, client_id)
+      props.put(kafka::CONNECTIONS_MAX_IDLE_MS_CONFIG, connections_max_idle_ms) unless connections_max_idle_ms.nil?
+      props.put(kafka::ENABLE_AUTO_COMMIT_CONFIG, enable_auto_commit)
+      props.put(kafka::EXCLUDE_INTERNAL_TOPICS_CONFIG, exclude_internal_topics) unless exclude_internal_topics.nil?
+      props.put(kafka::FETCH_MAX_BYTES_CONFIG, fetch_max_bytes) unless fetch_max_bytes.nil?
+      props.put(kafka::FETCH_MAX_WAIT_MS_CONFIG, fetch_max_wait_ms) unless fetch_max_wait_ms.nil?
+      props.put(kafka::FETCH_MIN_BYTES_CONFIG, fetch_min_bytes) unless fetch_min_bytes.nil?
+      props.put(kafka::GROUP_ID_CONFIG, group_id)
+      props.put(kafka::HEARTBEAT_INTERVAL_MS_CONFIG, heartbeat_interval_ms) unless heartbeat_interval_ms.nil?
+      props.put(kafka::KEY_DESERIALIZER_CLASS_CONFIG, key_deserializer_class)
+      props.put(kafka::MAX_PARTITION_FETCH_BYTES_CONFIG, max_partition_fetch_bytes) unless max_partition_fetch_bytes.nil?
+      props.put(kafka::MAX_POLL_RECORDS_CONFIG, max_poll_records) unless max_poll_records.nil?
+      props.put(kafka::MAX_POLL_INTERVAL_MS_CONFIG, max_poll_interval_ms) unless max_poll_interval_ms.nil?
+      props.put(kafka::METADATA_MAX_AGE_CONFIG, metadata_max_age_ms) unless metadata_max_age_ms.nil?
+      props.put(kafka::PARTITION_ASSIGNMENT_STRATEGY_CONFIG, partition_assignment_strategy) unless partition_assignment_strategy.nil?
+      props.put(kafka::RECEIVE_BUFFER_CONFIG, receive_buffer_bytes) unless receive_buffer_bytes.nil?
+      props.put(kafka::RECONNECT_BACKOFF_MS_CONFIG, reconnect_backoff_ms) unless reconnect_backoff_ms.nil?
+      props.put(kafka::REQUEST_TIMEOUT_MS_CONFIG, request_timeout_ms) unless request_timeout_ms.nil?
+      props.put(kafka::RETRY_BACKOFF_MS_CONFIG, retry_backoff_ms) unless retry_backoff_ms.nil?
+      props.put(kafka::SEND_BUFFER_CONFIG, send_buffer_bytes) unless send_buffer_bytes.nil?
+      props.put(kafka::SESSION_TIMEOUT_MS_CONFIG, session_timeout_ms) unless session_timeout_ms.nil?
+      props.put(kafka::VALUE_DESERIALIZER_CLASS_CONFIG, value_deserializer_class)
+
+      props.put("security.protocol", security_protocol) unless security_protocol.nil?
+
+      if security_protocol == "SSL"
+        set_trustore_keystore_config(props)
+      elsif security_protocol == "SASL_PLAINTEXT"
+        set_sasl_config(props)
+      elsif security_protocol == "SASL_SSL"
+        set_trustore_keystore_config(props)
+        set_sasl_config(props)
+      end
+
+      org.apache.kafka.clients.consumer.KafkaConsumer.new(props)
+    rescue => e
+      logger.error("Unable to create Kafka consumer from given configuration",
+                   :kafka_error_message => e,
+                   :cause => e.respond_to?(:getCause) ? e.getCause() : nil)
+      raise e
+    end
+  end
+
+  def set_trustore_keystore_config(props)
+    props.put("ssl.truststore.type", ssl_truststore_type) unless ssl_truststore_type.nil?
+    props.put("ssl.truststore.location", ssl_truststore_location) unless ssl_truststore_location.nil?
+    props.put("ssl.truststore.password", ssl_truststore_password.value) unless ssl_truststore_password.nil?
+
+    # Client auth stuff
+    props.put("ssl.keystore.type", ssl_keystore_type) unless ssl_keystore_type.nil?
+    props.put("ssl.key.password", ssl_key_password.value) unless ssl_key_password.nil?
+    props.put("ssl.keystore.location", ssl_keystore_location) unless ssl_keystore_location.nil?
+    props.put("ssl.keystore.password", ssl_keystore_password.value) unless ssl_keystore_password.nil?
+    props.put("ssl.endpoint.identification.algorithm", ssl_endpoint_identification_algorithm) unless ssl_endpoint_identification_algorithm.nil?
+  end
+
+  def set_sasl_config(props)
+    java.lang.System.setProperty("java.security.auth.login.config",jaas_path) unless jaas_path.nil?
+    java.lang.System.setProperty("java.security.krb5.conf",kerberos_config) unless kerberos_config.nil?
+
+    props.put("sasl.mechanism",sasl_mechanism)
+    if sasl_mechanism == "GSSAPI" && sasl_kerberos_service_name.nil?
+      raise LogStash::ConfigurationError, "sasl_kerberos_service_name must be specified when SASL mechanism is GSSAPI"
+    end
+
+    props.put("sasl.kerberos.service.name",sasl_kerberos_service_name) unless sasl_kerberos_service_name.nil?
+    props.put("sasl.jaas.config", sasl_jaas_config) unless sasl_jaas_config.nil?
+  end
+end #class LogStash::Inputs::Kafka