logstash-integration-kafka 10.3.0-java → 10.4.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cbbffe559ff38eaa26c9922c62a3722b2b1404446602238bcc7de309ca428da6
-  data.tar.gz: 0ce7e58400e06ae5e1c7f689a14220600efe7ad9ba46f285a52323a05568f69f
+  metadata.gz: 6ebbcd2d18d130e9fac997330c3c4b4bd9a959a982fe83215762b03638497ba4
+  data.tar.gz: 2b54ba231d9f74344a5ec321e0dcdec256ea5664001e7c3dc0323b2150761e30
 SHA512:
-  metadata.gz: 7be900607b24146853783d4e3a2e5a6d351853ab4dda0b30d10769adf8b13df21540cea2699d94660bd9e56580c9bf88e9684d5762ff7684f5d3a6804637b602
-  data.tar.gz: cd1f72db84b5af7de286d683f0f312e4b826c8d75b9b8fc475f693d8bf62b517c61950dd5d4887767f79f69df17b2c9bf1223e03880b2724333aa0b695b81525
+  metadata.gz: a8c2aa5c2123fa001f58fc3670bd90face614fed72cf24f17ad645ba4de3bd689923d51ba5b5dd3a9201507657f6ed54326ed48495d274bb7c2284525470bdf7
+  data.tar.gz: cebe4abeda34edd6d5d1872c96f1b119abfa7abb2e40c52fb061e2c0953789441223e4b5a93a0d2fd7e3de1918c592dce9f5fee91bb6713b0e16f167033c13ce

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 10.4.0
+ - added the input `isolation_level` to allow fine control of whether to return transactional messages [#44](https://github.com/logstash-plugins/logstash-integration-kafka/pull/44)
+
 ## 10.3.0
   - added the input and output `client_dns_lookup` parameter to allow control of how DNS requests are made
 

data/CONTRIBUTORS

@@ -12,6 +12,7 @@ Contributors:
 * Kurt Hurtado (kurtado)
 * Ry Biesemeyer (yaauie)
 * Rob Cowart (robcowart)
+* Tim te Beek (timtebeek)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/docs/input-kafka.asciidoc

@@ -95,6 +95,7 @@ See the https://kafka.apache.org/24/documentation for more details.
 | <<plugins-{type}s-{plugin}-fetch_min_bytes>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-group_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-heartbeat_interval_ms>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-isolation_level>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-kerberos_config>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-key_deserializer_class>> |<<string,string>>|No
@@ -315,6 +316,17 @@ consumers join or leave the group. The value must be set lower than
 `session.timeout.ms`, but typically should be set no higher than 1/3 of that value.
 It can be adjusted even lower to control the expected time for normal rebalances.
 
+[id="plugins-{type}s-{plugin}-isolation_level"]
+===== `isolation_level` 
+
+  * Value type is <<string,string>>
+  * Default value is `"read_uncommitted"`
+
+Controls how to read messages written transactionally. If set to `read_committed`, polling messages will only return
+transactional messages which have been committed. If set to `read_uncommitted` (the default), polling messages will
+return all messages, even transactional messages which have been aborted. Non-transactional messages will be returned
+unconditionally in either mode.
+
 [id="plugins-{type}s-{plugin}-jaas_path"]
 ===== `jaas_path` 
 

data/lib/logstash/inputs/kafka.rb

@@ -114,6 +114,11 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   # `session.timeout.ms`, but typically should be set no higher than 1/3 of that value.
   # It can be adjusted even lower to control the expected time for normal rebalances.
   config :heartbeat_interval_ms, :validate => :number, :default => 3000 # Kafka default
+  # Controls how to read messages written transactionally. If set to read_committed, consumer.poll()
+  # will only return transactional messages which have been committed. If set to read_uncommitted'
+  # (the default), consumer.poll() will return all messages, even transactional messages which have
+  # been aborted. Non-transactional messages will be returned unconditionally in either mode.
+  config :isolation_level, :validate => ["read_uncommitted", "read_committed"], :default => "read_uncommitted" # Kafka default
   # Java Class used to deserialize the record's key
   config :key_deserializer_class, :validate => :string, :default => "org.apache.kafka.common.serialization.StringDeserializer"
   # The maximum delay between invocations of poll() when using consumer group management. This places 
@@ -311,6 +316,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
       props.put(kafka::FETCH_MIN_BYTES_CONFIG, fetch_min_bytes.to_s) unless fetch_min_bytes.nil?
       props.put(kafka::GROUP_ID_CONFIG, group_id)
       props.put(kafka::HEARTBEAT_INTERVAL_MS_CONFIG, heartbeat_interval_ms.to_s) unless heartbeat_interval_ms.nil?
+      props.put(kafka::ISOLATION_LEVEL_CONFIG, isolation_level)
       props.put(kafka::KEY_DESERIALIZER_CLASS_CONFIG, key_deserializer_class)
       props.put(kafka::MAX_PARTITION_FETCH_BYTES_CONFIG, max_partition_fetch_bytes.to_s) unless max_partition_fetch_bytes.nil?
       props.put(kafka::MAX_POLL_RECORDS_CONFIG, max_poll_records.to_s) unless max_poll_records.nil?

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '10.3.0'
+  s.version         = '10.4.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-kafka
 version: !ruby/object:Gem::Version
-  version: 10.3.0
+  version: 10.4.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-06-18 00:00:00.000000000 Z
+date: 2020-07-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement