logstash-input-kafka 6.1.0 → 6.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ca18508e563e481b0e21452577fd2ab764126df5
-  data.tar.gz: f5d951ba0229244db1a08eae618c99caea32b9e9
+  metadata.gz: b03d2e1ba09c2b8d379222908f610340a86d495c
+  data.tar.gz: 31dce488f638fc6b32ef72c222979dfef845c033
 SHA512:
-  metadata.gz: 92df3686bcac5a2056acaf6ae59fe6ee185174568250d298e1158678708ff9da47700b85600b301f31435c550411c6495d838499009fbc088da826aa62ff33ea
-  data.tar.gz: 5675c2530563e407aec9bd3a15554b9644d4d9f393b6c5d9ac8e1b49985c27439415a4468704ff204890bef124fb92d88b09553a6e87ea569f0ef9cc5a47b64f
+  metadata.gz: 45bb9dba8b2a3d168c3466c47263cd084962e3e518872a33580856ee7dfd71ec7110693190ac5ef15328632dae9c7df28aa8d36985ccfc45498f3fece58295ae
+  data.tar.gz: 3fa72f812e3084d04c1f311d271f63a89637777b553f13b5563b53486fec728d1a975ed95a3a41b1bd191912335ff05fd5529113fe8cf2dc30b6c4261aafde1e

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 6.2.0
+  - Expose config `max_poll_interval_ms` to allow consumer to send heartbeats from a background thread
+  - Expose config `fetch_max_bytes` to control client's fetch response size limit
+
 ## 6.1.0
   - Add Kerberos authentication support.
 

data/lib/logstash/inputs/kafka.rb

@@ -14,10 +14,10 @@ require 'logstash-input-kafka_jars.rb'
 # |==========================================================
 # |Kafka Client Version |Logstash Version |Plugin Version |Security Features |Why?
 # |0.8       |2.0.0 - 2.x.x   |<3.0.0 | |Legacy, 0.8 is still popular 
-# |0.9       |2.0.0 - 2.3.x   | 3.x.x |Basic Auth, SSL |Works with the old Ruby Event API (`event['product']['price'] = 10`)  
-# |0.9       |2.4.0 - 5.0.x   | 4.x.x |Basic Auth, SSL |Works with the new getter/setter APIs (`event.set('[product][price]', 10)`)
-# |0.10.0.x  |2.4.0 - 5.0.x   | 5.x.x |Basic Auth, SSL |Not compatible with the <= 0.9 broker
-# |0.10.1.x  |2.4.0 - 5.0.x   | 6.x.x |Basic Auth, SSL |Not compatible with <= 0.10.0.x broker
+# |0.9       |2.0.0 - 2.3.x   | 3.x.x |SSL |Works with the old Ruby Event API (`event['product']['price'] = 10`)  
+# |0.9       |2.4.0 - 5.0.x   | 4.x.x |SSL |Works with the new getter/setter APIs (`event.set('[product][price]', 10)`)
+# |0.10.0.x  |2.4.0 - 5.0.x   | 5.x.x |SSL |Not compatible with the <= 0.9 broker
+# |0.10.1.x  |2.4.0 - 5.0.x   | 6.x.x |SSL |Not compatible with <= 0.10.0.x broker
 # |==========================================================
 # 
 # NOTE: We recommended that you use matching Kafka client and broker versions. During upgrades, you should
@@ -83,6 +83,10 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   # Whether records from internal topics (such as offsets) should be exposed to the consumer.
   # If set to true the only way to receive records from an internal topic is subscribing to it.
   config :exclude_internal_topics, :validate => :string
+  # The maximum amount of data the server should return for a fetch request. This is not an 
+  # absolute maximum, if the first message in the first non-empty partition of the fetch is larger 
+  # than this value, the message will still be returned to ensure that the consumer can make progress.
+  config :fetch_max_bytes, :validate => :string
   # The maximum amount of time the server will block before answering the fetch request if
   # there isn't sufficient data to immediately satisfy `fetch_min_bytes`. This
   # should be less than or equal to the timeout used in `poll_timeout_ms`
@@ -103,6 +107,12 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   config :heartbeat_interval_ms, :validate => :string
   # Java Class used to deserialize the record's key
   config :key_deserializer_class, :validate => :string, :default => "org.apache.kafka.common.serialization.StringDeserializer"
+  # The maximum delay between invocations of poll() when using consumer group management. This places 
+  # an upper bound on the amount of time that the consumer can be idle before fetching more records. 
+  # If poll() is not called before expiration of this timeout, then the consumer is considered failed and 
+  # the group will rebalance in order to reassign the partitions to another member.
+  # The value of the configuration `request_timeout_ms` must always be larger than max_poll_interval_ms
+  config :max_poll_interval_ms, :validate => :string
   # The maximum amount of data per-partition the server will return. The maximum total memory used for a
   # request will be <code>#partitions * max.partition.fetch.bytes</code>. This size must be at least
   # as large as the maximum message size the server allows or else it is possible for the producer to
@@ -270,6 +280,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
       props.put(kafka::CONNECTIONS_MAX_IDLE_MS_CONFIG, connections_max_idle_ms) unless connections_max_idle_ms.nil?
       props.put(kafka::ENABLE_AUTO_COMMIT_CONFIG, enable_auto_commit)
       props.put(kafka::EXCLUDE_INTERNAL_TOPICS_CONFIG, exclude_internal_topics) unless exclude_internal_topics.nil?
+      props.put(kafka::FETCH_MAX_BYTES_CONFIG, fetch_max_bytes) unless fetch_max_bytes.nil?
       props.put(kafka::FETCH_MAX_WAIT_MS_CONFIG, fetch_max_wait_ms) unless fetch_max_wait_ms.nil?
       props.put(kafka::FETCH_MIN_BYTES_CONFIG, fetch_min_bytes) unless fetch_min_bytes.nil?
       props.put(kafka::GROUP_ID_CONFIG, group_id)
@@ -277,6 +288,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
       props.put(kafka::KEY_DESERIALIZER_CLASS_CONFIG, key_deserializer_class)
       props.put(kafka::MAX_PARTITION_FETCH_BYTES_CONFIG, max_partition_fetch_bytes) unless max_partition_fetch_bytes.nil?
       props.put(kafka::MAX_POLL_RECORDS_CONFIG, max_poll_records) unless max_poll_records.nil?
+      props.put(kafka::MAX_POLL_INTERVAL_MS_CONFIG, max_poll_interval_ms) unless max_poll_interval_ms.nil?
       props.put(kafka::METADATA_MAX_AGE_MS_CONFIG, metadata_max_age_ms) unless metadata_max_age_ms.nil?
       props.put(kafka::PARTITION_ASSIGNMENT_STRATEGY_CONFIG, partition_assignment_strategy) unless partition_assignment_strategy.nil?
       props.put(kafka::RECEIVE_BUFFER_CONFIG, receive_buffer_bytes) unless receive_buffer_bytes.nil?

data/logstash-input-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-kafka'
-  s.version         = '6.1.0'
+  s.version         = '6.2.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = 'This input will read events from a Kafka topic. It uses the high level consumer API provided by Kafka to read messages from the broker'
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-kafka
 version: !ruby/object:Gem::Version
-  version: 6.1.0
+  version: 6.2.0
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-11-17 00:00:00.000000000 Z
+date: 2016-12-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement