logstash-input-kafka 6.2.7 → 6.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: adf059fe901952446cc470414f3bb55003b44aeb
-  data.tar.gz: af25a62efc3f16b61742956de53e162fa19669c4
+  metadata.gz: 67d7e39487de32bc46086f71dc728d0b73a8d51d
+  data.tar.gz: 3e95c206a4142c3c90aa814e33d534edf5a6ce94
 SHA512:
-  metadata.gz: d8646d1e9f74ba8ff431897ec9be8eb6dded0dc86f4bbec135d7eb8b4e03f68f62bbc4b6fc1b2de1fedea874847d4b46e86e4f6fc4064db1c84cc50941f3ee31
-  data.tar.gz: c3918567af917e00203ae5a8b49855e2939ecdd6f132b4c25ae6c97abeaadac5454b31117ab61ef9103ae753d76c131abe73e1509a153eeea8448d20769994b3
+  metadata.gz: aba387cb731bc155f245f6fd913b4cdd41b0cd9ef39468f392412fb4424b9474bf01f4156465115eb94860d1a22cf8c60a2dcae81be699b30a0d91ce5ba4bb70
+  data.tar.gz: aaebae3fb793842fca36c652c3de82812bbb6811a82d7d3ad6e29ebb3c9de38ebf14fd40e6eb5c14f37ac89b39d244de4991bcb6c605de055810da71094212d7

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 6.3.0
+  - Upgrade Kafka client to version 0.10.2.1
+
 ## 6.2.7
   - Fix NPE when SASL_SSL+PLAIN (no Kerberos) is specified.
 

data/docs/index.asciidoc

@@ -0,0 +1,551 @@
+:plugin: kafka
+:type: input
+
+///////////////////////////////////////////
+START - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+:version: %VERSION%
+:release_date: %RELEASE_DATE%
+:changelog_url: %CHANGELOG_URL%
+:include_path: ../../../logstash/docs/include
+///////////////////////////////////////////
+END - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+
+[id="plugins-{type}-{plugin}"]
+
+=== Kafka
+
+include::{include_path}/plugin_header.asciidoc[]
+
+==== Description
+
+This input will read events from a Kafka topic. It uses the 0.10 version of
+the consumer API provided by Kafka to read messages from the broker.
+
+Here's a compatibility matrix that shows the Kafka client versions that are compatible with each combination
+of Logstash and the Kafka input plugin: 
+
+[options="header"]
+|==========================================================
+|Kafka Client Version |Logstash Version |Plugin Version |Why?
+|0.8       |2.0.0 - 2.x.x   |<3.0.0 |Legacy, 0.8 is still popular 
+|0.9       |2.0.0 - 2.3.x   | 3.x.x |Works with the old Ruby Event API (`event['product']['price'] = 10`)  
+|0.9       |2.4.x - 5.x.x   | 4.x.x |Works with the new getter/setter APIs (`event.set('[product][price]', 10)`)
+|0.10.0.x  |2.4.x - 5.x.x   | 5.x.x |Not compatible with the <= 0.9 broker
+|0.10.1.x  |2.4.x - 5.x.x   | 6.x.x |
+|==========================================================
+
+NOTE: We recommended that you use matching Kafka client and broker versions. During upgrades, you should
+upgrade brokers before clients because brokers target backwards compatibility. For example, the 0.9 broker
+is compatible with both the 0.8 consumer and 0.9 consumer APIs, but not the other way around.
+
+This input supports connecting to Kafka over:
+
+* SSL (requires plugin version 3.0.0 or later)
+* Kerberos SASL (requires plugin version 5.1.0 or later) 
+
+By default security is disabled but can be turned on as needed.
+
+The Logstash Kafka consumer handles group management and uses the default offset management
+strategy using Kafka topics.
+
+Logstash instances by default form a single logical group to subscribe to Kafka topics
+Each Logstash Kafka consumer can run multiple threads to increase read throughput. Alternatively, 
+you could run multiple Logstash instances with the same `group_id` to spread the load across
+physical machines. Messages in a topic will be distributed to all Logstash instances with
+the same `group_id`.
+
+Ideally you should have as many threads as the number of partitions for a perfect balance --
+more threads than partitions means that some threads will be idle
+
+For more information see http://kafka.apache.org/documentation.html#theconsumer
+
+Kafka consumer configuration: http://kafka.apache.org/documentation.html#consumerconfigs
+
+
+[id="plugins-{type}s-{plugin}-options"]
+==== Kafka Input Configuration Options
+
+This plugin supports the following configuration options plus the <<plugins-{type}s-common-options>> described later.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-auto_commit_interval_ms>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-auto_offset_reset>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-bootstrap_servers>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-check_crcs>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-client_id>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-connections_max_idle_ms>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-consumer_threads>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-decorate_events>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-enable_auto_commit>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-exclude_internal_topics>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-fetch_max_bytes>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-fetch_max_wait_ms>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-fetch_min_bytes>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-group_id>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-heartbeat_interval_ms>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-key_deserializer_class>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-max_partition_fetch_bytes>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-max_poll_interval_ms>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-max_poll_records>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-metadata_max_age_ms>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-partition_assignment_strategy>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-poll_timeout_ms>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-receive_buffer_bytes>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-reconnect_backoff_ms>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-request_timeout_ms>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-retry_backoff_ms>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-sasl_mechanism>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-security_protocol>> |<<string,string>>, one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No
+| <<plugins-{type}s-{plugin}-send_buffer_bytes>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-session_timeout_ms>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-ssl_key_password>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-ssl_keystore_location>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-ssl_keystore_password>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-ssl_keystore_type>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-ssl_truststore_location>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-ssl_truststore_password>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-ssl_truststore_type>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-topics>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-topics_pattern>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-value_deserializer_class>> |<<string,string>>|No
+|=======================================================================
+
+Also see <<plugins-{type}s-common-options>> for a list of options supported by all
+input plugins.
+
+&nbsp;
+
+[id="plugins-{type}s-{plugin}-auto_commit_interval_ms"]
+===== `auto_commit_interval_ms` 
+
+  * Value type is <<string,string>>
+  * Default value is `"5000"`
+
+The frequency in milliseconds that the consumer offsets are committed to Kafka.
+
+[id="plugins-{type}s-{plugin}-auto_offset_reset"]
+===== `auto_offset_reset` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+What to do when there is no initial offset in Kafka or if an offset is out of range:
+
+* earliest: automatically reset the offset to the earliest offset
+* latest: automatically reset the offset to the latest offset
+* none: throw exception to the consumer if no previous offset is found for the consumer's group
+* anything else: throw exception to the consumer.
+
+[id="plugins-{type}s-{plugin}-bootstrap_servers"]
+===== `bootstrap_servers` 
+
+  * Value type is <<string,string>>
+  * Default value is `"localhost:9092"`
+
+A list of URLs of Kafka instances to use for establishing the initial connection to the cluster.
+This list should be in the form of `host1:port1,host2:port2` These urls are just used
+for the initial connection to discover the full cluster membership (which may change dynamically)
+so this list need not contain the full set of servers (you may want more than one, though, in
+case a server is down).
+
+[id="plugins-{type}s-{plugin}-check_crcs"]
+===== `check_crcs` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Automatically check the CRC32 of the records consumed. This ensures no on-the-wire or on-disk
+corruption to the messages occurred. This check adds some overhead, so it may be
+disabled in cases seeking extreme performance.
+
+[id="plugins-{type}s-{plugin}-client_id"]
+===== `client_id` 
+
+  * Value type is <<string,string>>
+  * Default value is `"logstash"`
+
+The id string to pass to the server when making requests. The purpose of this
+is to be able to track the source of requests beyond just ip/port by allowing
+a logical application name to be included.
+
+[id="plugins-{type}s-{plugin}-connections_max_idle_ms"]
+===== `connections_max_idle_ms` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Close idle connections after the number of milliseconds specified by this config.
+
+[id="plugins-{type}s-{plugin}-consumer_threads"]
+===== `consumer_threads` 
+
+  * Value type is <<number,number>>
+  * Default value is `1`
+
+Ideally you should have as many threads as the number of partitions for a perfect
+balance — more threads than partitions means that some threads will be idle
+
+[id="plugins-{type}s-{plugin}-decorate_events"]
+===== `decorate_events` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+Option to add Kafka metadata like topic, message size to the event.
+This will add a field named `kafka` to the logstash event containing the following attributes:
+  `topic`: The topic this message is associated with
+  `consumer_group`: The consumer group used to read in this event
+  `partition`: The partition this message is associated with
+  `offset`: The offset from the partition this message is associated with
+  `key`: A ByteBuffer containing the message key
+
+[id="plugins-{type}s-{plugin}-enable_auto_commit"]
+===== `enable_auto_commit` 
+
+  * Value type is <<string,string>>
+  * Default value is `"true"`
+
+If true, periodically commit to Kafka the offsets of messages already returned by the consumer. 
+This committed offset will be used when the process fails as the position from
+which the consumption will begin.
+
+[id="plugins-{type}s-{plugin}-exclude_internal_topics"]
+===== `exclude_internal_topics` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Whether records from internal topics (such as offsets) should be exposed to the consumer.
+If set to true the only way to receive records from an internal topic is subscribing to it.
+
+[id="plugins-{type}s-{plugin}-fetch_max_bytes"]
+===== `fetch_max_bytes` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The maximum amount of data the server should return for a fetch request. This is not an 
+absolute maximum, if the first message in the first non-empty partition of the fetch is larger 
+than this value, the message will still be returned to ensure that the consumer can make progress.
+
+[id="plugins-{type}s-{plugin}-fetch_max_wait_ms"]
+===== `fetch_max_wait_ms` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The maximum amount of time the server will block before answering the fetch request if
+there isn't sufficient data to immediately satisfy `fetch_min_bytes`. This
+should be less than or equal to the timeout used in `poll_timeout_ms`
+
+[id="plugins-{type}s-{plugin}-fetch_min_bytes"]
+===== `fetch_min_bytes` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The minimum amount of data the server should return for a fetch request. If insufficient
+data is available the request will wait for that much data to accumulate
+before answering the request.
+
+[id="plugins-{type}s-{plugin}-group_id"]
+===== `group_id` 
+
+  * Value type is <<string,string>>
+  * Default value is `"logstash"`
+
+The identifier of the group this consumer belongs to. Consumer group is a single logical subscriber
+that happens to be made up of multiple processors. Messages in a topic will be distributed to all
+Logstash instances with the same `group_id`
+
+[id="plugins-{type}s-{plugin}-heartbeat_interval_ms"]
+===== `heartbeat_interval_ms` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The expected time between heartbeats to the consumer coordinator. Heartbeats are used to ensure 
+that the consumer's session stays active and to facilitate rebalancing when new
+consumers join or leave the group. The value must be set lower than
+`session.timeout.ms`, but typically should be set no higher than 1/3 of that value.
+It can be adjusted even lower to control the expected time for normal rebalances.
+
+[id="plugins-{type}s-{plugin}-jaas_path"]
+===== `jaas_path` 
+
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
+
+The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization 
+services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client:
+[source,java]
+----------------------------------
+KafkaClient {
+  com.sun.security.auth.module.Krb5LoginModule required
+  useTicketCache=true
+  renewTicket=true
+  serviceName="kafka";
+  };
+----------------------------------
+
+Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these  
+to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same 
+`jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on 
+different JVM instances.
+
+[id="plugins-{type}s-{plugin}-kerberos_config"]
+===== `kerberos_config` 
+
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
+
+Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html
+
+[id="plugins-{type}s-{plugin}-key_deserializer_class"]
+===== `key_deserializer_class` 
+
+  * Value type is <<string,string>>
+  * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"`
+
+Java Class used to deserialize the record's key
+
+[id="plugins-{type}s-{plugin}-max_partition_fetch_bytes"]
+===== `max_partition_fetch_bytes` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The maximum amount of data per-partition the server will return. The maximum total memory used for a
+request will be <code>#partitions * max.partition.fetch.bytes</code>. This size must be at least
+as large as the maximum message size the server allows or else it is possible for the producer to
+send messages larger than the consumer can fetch. If that happens, the consumer can get stuck trying
+to fetch a large message on a certain partition.
+
+[id="plugins-{type}s-{plugin}-max_poll_interval_ms"]
+===== `max_poll_interval_ms` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The maximum delay between invocations of poll() when using consumer group management. This places 
+an upper bound on the amount of time that the consumer can be idle before fetching more records. 
+If poll() is not called before expiration of this timeout, then the consumer is considered failed and 
+the group will rebalance in order to reassign the partitions to another member.
+The value of the configuration `request_timeout_ms` must always be larger than max_poll_interval_ms
+
+[id="plugins-{type}s-{plugin}-max_poll_records"]
+===== `max_poll_records` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The maximum number of records returned in a single call to poll().
+
+[id="plugins-{type}s-{plugin}-metadata_max_age_ms"]
+===== `metadata_max_age_ms` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The period of time in milliseconds after which we force a refresh of metadata even if
+we haven't seen any partition leadership changes to proactively discover any new brokers or partitions
+
+[id="plugins-{type}s-{plugin}-partition_assignment_strategy"]
+===== `partition_assignment_strategy` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The class name of the partition assignment strategy that the client will use to distribute
+partition ownership amongst consumer instances
+
+[id="plugins-{type}s-{plugin}-poll_timeout_ms"]
+===== `poll_timeout_ms` 
+
+  * Value type is <<number,number>>
+  * Default value is `100`
+
+Time kafka consumer will wait to receive new messages from topics
+
+[id="plugins-{type}s-{plugin}-receive_buffer_bytes"]
+===== `receive_buffer_bytes` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The size of the TCP receive buffer (SO_RCVBUF) to use when reading data.
+
+[id="plugins-{type}s-{plugin}-reconnect_backoff_ms"]
+===== `reconnect_backoff_ms` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The amount of time to wait before attempting to reconnect to a given host.
+This avoids repeatedly connecting to a host in a tight loop.
+This backoff applies to all requests sent by the consumer to the broker.
+
+[id="plugins-{type}s-{plugin}-request_timeout_ms"]
+===== `request_timeout_ms` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The configuration controls the maximum amount of time the client will wait
+for the response of a request. If the response is not received before the timeout
+elapses the client will resend the request if necessary or fail the request if
+retries are exhausted.
+
+[id="plugins-{type}s-{plugin}-retry_backoff_ms"]
+===== `retry_backoff_ms` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The amount of time to wait before attempting to retry a failed fetch request
+to a given topic partition. This avoids repeated fetching-and-failing in a tight loop.
+
+[id="plugins-{type}s-{plugin}-sasl_kerberos_service_name"]
+===== `sasl_kerberos_service_name` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The Kerberos principal name that Kafka broker runs as. 
+This can be defined either in Kafka's JAAS config or in Kafka's config.
+
+[id="plugins-{type}s-{plugin}-sasl_mechanism"]
+===== `sasl_mechanism` 
+
+  * Value type is <<string,string>>
+  * Default value is `"GSSAPI"`
+
+http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. 
+This may be any mechanism for which a security provider is available.
+GSSAPI is the default mechanism.
+
+[id="plugins-{type}s-{plugin}-security_protocol"]
+===== `security_protocol` 
+
+  * Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL`
+  * Default value is `"PLAINTEXT"`
+
+Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL
+
+[id="plugins-{type}s-{plugin}-send_buffer_bytes"]
+===== `send_buffer_bytes` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The size of the TCP send buffer (SO_SNDBUF) to use when sending data
+
+[id="plugins-{type}s-{plugin}-session_timeout_ms"]
+===== `session_timeout_ms` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The timeout after which, if the `poll_timeout_ms` is not invoked, the consumer is marked dead
+and a rebalance operation is triggered for the group identified by `group_id`
+
+[id="plugins-{type}s-{plugin}-ssl"]
+===== `ssl`  (DEPRECATED)
+
+  * DEPRECATED WARNING: This configuration item is deprecated and may not be available in future versions.
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+Enable SSL/TLS secured communication to Kafka broker.
+
+[id="plugins-{type}s-{plugin}-ssl_key_password"]
+===== `ssl_key_password` 
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+The password of the private key in the key store file.
+
+[id="plugins-{type}s-{plugin}-ssl_keystore_location"]
+===== `ssl_keystore_location` 
+
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
+
+If client authentication is required, this setting stores the keystore path.
+
+[id="plugins-{type}s-{plugin}-ssl_keystore_password"]
+===== `ssl_keystore_password` 
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+If client authentication is required, this setting stores the keystore password
+
+[id="plugins-{type}s-{plugin}-ssl_keystore_type"]
+===== `ssl_keystore_type` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The keystore type.
+
+[id="plugins-{type}s-{plugin}-ssl_truststore_location"]
+===== `ssl_truststore_location` 
+
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
+
+The JKS truststore path to validate the Kafka broker's certificate.
+
+[id="plugins-{type}s-{plugin}-ssl_truststore_password"]
+===== `ssl_truststore_password` 
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+The truststore password
+
+[id="plugins-{type}s-{plugin}-ssl_truststore_type"]
+===== `ssl_truststore_type` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The truststore type.
+
+[id="plugins-{type}s-{plugin}-topics"]
+===== `topics` 
+
+  * Value type is <<array,array>>
+  * Default value is `["logstash"]`
+
+A list of topics to subscribe to, defaults to ["logstash"].
+
+[id="plugins-{type}s-{plugin}-topics_pattern"]
+===== `topics_pattern` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+A topic regex pattern to subscribe to. 
+The topics configuration will be ignored when using this configuration.
+
+[id="plugins-{type}s-{plugin}-value_deserializer_class"]
+===== `value_deserializer_class` 
+
+  * Value type is <<string,string>>
+  * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"`
+
+Java Class used to deserialize the record's value
+
+
+
+include::{include_path}/{type}.asciidoc[]

data/lib/logstash/inputs/kafka.rb

@@ -61,7 +61,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   # * none: throw exception to the consumer if no previous offset is found for the consumer's group
   # * anything else: throw exception to the consumer.
   config :auto_offset_reset, :validate => :string
-  # A list of URLs to use for establishing the initial connection to the cluster.
+  # A list of URLs of Kafka instances to use for establishing the initial connection to the cluster.
   # This list should be in the form of `host1:port1,host2:port2` These urls are just used
   # for the initial connection to discover the full cluster membership (which may change dynamically)
   # so this list need not contain the full set of servers (you may want more than one, though, in
@@ -261,6 +261,10 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
               logstash_queue << event
             end
           end
+          # Manual offset commit
+          if @enable_auto_commit == "false"
+            consumer.commitSync
+          end
         end
       rescue org.apache.kafka.common.errors.WakeupException => e
         raise e if !stop?

data/logstash-input-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-kafka'
-  s.version         = '6.2.7'
+  s.version         = '6.3.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = 'This input will read events from a Kafka topic. It uses the high level consumer API provided by Kafka to read messages from the broker'
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -10,7 +10,7 @@ Gem::Specification.new do |s|
   s.require_paths = ['lib']
 
   # Files
-  s.files = Dir['lib/**/*.rb','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
@@ -18,7 +18,7 @@ Gem::Specification.new do |s|
   # Special flag to let us know this is actually a logstash plugin
   s.metadata = { 'logstash_plugin' => 'true', 'group' => 'input'}
 
-  s.requirements << "jar 'org.apache.kafka:kafka-clients', '0.10.1.1'"
+  s.requirements << "jar 'org.apache.kafka:kafka-clients', '0.10.2.1'"
   s.requirements << "jar 'org.slf4j:slf4j-log4j12', '1.7.21'"
   s.requirements << "jar 'org.apache.logging.log4j:log4j-1.2-api', '2.6.2'"
 

data/spec/integration/inputs/kafka_spec.rb

@@ -17,6 +17,7 @@ describe "inputs/kafka", :integration => true do
   let(:lz4_config) { { 'topics' => ['logstash_topic_lz4'], 'codec' => 'plain', 'group_id' => group_id_1, 'auto_offset_reset' => 'earliest'} }
   let(:pattern_config) { { 'topics_pattern' => 'logstash_topic_.*', 'group_id' => group_id_2, 'codec' => 'plain', 'auto_offset_reset' => 'earliest'} }  
   let(:decorate_config) { { 'topics' => ['logstash_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_3, 'auto_offset_reset' => 'earliest', 'decorate_events' => true} }
+  let(:manual_commit_config) { { 'topics' => ['logstash_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_4, 'auto_offset_reset' => 'earliest', 'enable_auto_commit' => 'false'} }
   let(:timeout_seconds) { 120 }
   let(:num_events) { 103 }
 
@@ -109,4 +110,23 @@ describe "inputs/kafka", :integration => true do
       expect(event.get("kafka")["consumer_group"]).to eq(group_id_3)
     end
   end
+
+  describe "#kafka-offset-commit" do
+    def thread_it(kafka_input, queue)
+      Thread.new do
+        begin
+          kafka_input.run(queue)
+        end
+      end
+    end
+
+    it "should manually commit offsets" do
+      kafka_input = LogStash::Inputs::Kafka.new(manual_commit_config)
+      queue = Array.new
+      t = thread_it(kafka_input, queue)
+      t.run
+      wait(timeout_seconds).for { queue.length }.to eq(num_events)
+      expect(queue.length).to eq(num_events)
+    end
+  end
 end

metadata

@@ -1,39 +1,30 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-kafka
 version: !ruby/object:Gem::Version
-  version: 6.2.7
+  version: 6.3.0
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-04-09 00:00:00.000000000 Z
+date: 2017-05-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: jar-dependencies
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.3.2
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.3.2
+  name: jar-dependencies
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: logstash-core-plugin-api
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '1.60'
-    - - "<="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.99'
+        version: 0.3.2
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -42,46 +33,46 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
+  name: logstash-core-plugin-api
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-codec-json
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-codec-json
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-codec-plain
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-codec-plain
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: stud
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.0.22
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: 0.1.0
+        version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -90,36 +81,45 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 0.1.0
+  name: stud
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.0.22
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-devutils
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: rspec-wait
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: rspec-wait
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
@@ -133,12 +133,22 @@ files:
 - LICENSE
 - NOTICE.TXT
 - README.md
+- docs/index.asciidoc
+- lib/log4j/log4j/1.2.17/log4j-1.2.17.jar
 - lib/logstash-input-kafka_jars.rb
 - lib/logstash/inputs/kafka.rb
+- lib/net/jpountz/lz4/lz4/1.3.0/lz4-1.3.0.jar
+- lib/org/apache/kafka/kafka-clients/0.10.2.1/kafka-clients-0.10.2.1.jar
+- lib/org/apache/logging/log4j/log4j-1.2-api/2.6.2/log4j-1.2-api-2.6.2.jar
+- lib/org/apache/logging/log4j/log4j-api/2.6.2/log4j-api-2.6.2.jar
+- lib/org/apache/logging/log4j/log4j-core/2.6.2/log4j-core-2.6.2.jar
+- lib/org/slf4j/slf4j-api/1.7.21/slf4j-api-1.7.21.jar
+- lib/org/slf4j/slf4j-log4j12/1.7.21/slf4j-log4j12-1.7.21.jar
+- lib/org/xerial/snappy/snappy-java/1.1.2.6/snappy-java-1.1.2.6.jar
 - logstash-input-kafka.gemspec
 - spec/integration/inputs/kafka_spec.rb
 - spec/unit/inputs/kafka_spec.rb
-- vendor/jar-dependencies/runtime-jars/kafka-clients-0.10.1.1.jar
+- vendor/jar-dependencies/runtime-jars/kafka-clients-0.10.2.1.jar
 - vendor/jar-dependencies/runtime-jars/log4j-1.2-api-2.6.2.jar
 - vendor/jar-dependencies/runtime-jars/log4j-1.2.17.jar
 - vendor/jar-dependencies/runtime-jars/log4j-api-2.6.2.jar
@@ -168,7 +178,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements:
-- jar 'org.apache.kafka:kafka-clients', '0.10.1.1'
+- jar 'org.apache.kafka:kafka-clients', '0.10.2.1'
 - jar 'org.slf4j:slf4j-log4j12', '1.7.21'
 - jar 'org.apache.logging.log4j:log4j-1.2-api', '2.6.2'
 rubyforge_project: