logstash-integration-kafka 12.0.5-java → 12.1.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9aabb58fee374e97f9acb560f65340e1bd06303a60e39a010113980099dc51e8
-  data.tar.gz: be2acb673a6a9d463d709814bbf93fabc4089dc9d5f96aa4266cd3a18c0208cd
+  metadata.gz: 7f5d9772f13ac08019642f4ead686f5bf0b0cbfef80c7a9ecb4115cd0c961051
+  data.tar.gz: '0935e12f42e2aa0d28725b41d16f9705e98edafe8d1977e62bf248f7f52912e4'
 SHA512:
-  metadata.gz: 95bd3afc976010cc01bd4ac9e8d6844631f28cadbf5f45121c968bc9d3abd8475dbc7c78f99cda20b16349f84404a88a9fc121f7513bf8fa7baa37988b2d74c0
-  data.tar.gz: 75cc2e4b39d89303dd5b4927c5f8ea1008902f30e79bf8dcc9ac20d774d43020b705237d1c1510cd3eb949fc8832b601c51b08e21edd7b7ce3022262154083c0
+  metadata.gz: 43a3515e8aa880cf55e5d94d114c42f8a91c2c702cc4e4ed472e61fe313c7d589e2c048c24844003c1be953bcebaab6cc3f06001a7a87a780b71ecb6ede4cf3d
+  data.tar.gz: 74bda533606959d6631ae5b18ed81aa2165b69880adb31038ae5dfe84199a80994c234c759b616e7d575fac1649d55b7da08ba7daf59bc0ef003a144f33dc338

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 12.1.0
+  - Update Kafka client to 4.2.0 [#243](https://github.com/logstash-plugins/logstash-integration-kafka/pull/243)
+    - Remove explicit `lz4-java` dependency (now transitive from Kafka client)
+    - Document `by_duration` offset reset strategy (available since Apache Kafka 4.0.0)
+
+## 12.0.6
+  - [DOC] Add info about Kafka timestamp behavior [#240](https://github.com/logstash-plugins/logstash-integration-kafka/pull/240)
+
 ## 12.0.5
   - Redact `sasl_jaas_config` to prevent credentials from appearing in debug logs. [#232](https://github.com/logstash-plugins/logstash-integration-kafka/pull/232)
 
@@ -18,7 +26,7 @@
     - Breaking Change: partitioner options `default` and `uniform_sticky` are removed
     - `linger_ms` default value changed from 0 to 5
   - Add `group_protocols` options for configuring Kafka consumer rebalance protocol
-    - Setting `group_protocols => consumer` opts in to the new consumer group protocol
+    - Setting `group_protocol => consumer` opts in to the new consumer group protocol
 
 ## 11.7.0
   - Add `reconnect_backoff_max_ms` option for configuring kafka client [#204](https://github.com/logstash-plugins/logstash-integration-kafka/pull/204)

data/docs/index.asciidoc

@@ -1,7 +1,7 @@
 :plugin: kafka
 :type: integration
 :no_codec:
-:kafka_client: 4.1.0
+:kafka_client: 4.2.0
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!

data/docs/input-kafka.asciidoc

@@ -2,8 +2,8 @@
 :plugin: kafka
 :type: input
 :default_codec: plain
-:kafka_client: 4.1.0
-:kafka_client_doc: 41
+:kafka_client: 4.2.0
+:kafka_client_doc: 42
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!
@@ -211,6 +211,7 @@ What to do when there is no initial offset in Kafka or if an offset is out of ra
 
 * earliest: automatically reset the offset to the earliest offset
 * latest: automatically reset the offset to the latest offset
+* by_duration:<duration>: automatically reset the offset to a configured duration from the current timestamp. The duration must be specified in ISO 8601 format (PnDTnHnMn.nS). Negative durations are not allowed. For example, `by_duration:PT1H` resets to 1 hour ago. Available since Apache Kafka 4.0.0.
 * none: throw exception to the consumer if no previous offset is found for the consumer's group
 * anything else: throw exception to the consumer.
 

data/docs/output-kafka.asciidoc

@@ -2,8 +2,8 @@
 :plugin: kafka
 :type: output
 :default_codec: plain
-:kafka_client: 4.1.0
-:kafka_client_doc: 41
+:kafka_client: 4.2.0
+:kafka_client_doc: 42
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!
@@ -66,6 +66,24 @@ https://kafka.apache.org/{kafka_client_doc}/documentation.html#producerconfigs
 
 NOTE:  This plugin does not support using a proxy when communicating to the Kafka broker.
 
+.Kafka timestamps and Logstash
+****
+* Kafka 3.6+ introduces stricter timestamp validation with the introduction of two new broker/topic-level properties: https://docs.confluent.io/platform/current/installation/configuration/topic-configs.html#message-timestamp-before-max-ms[log.message.timestamp.before.max.ms] and 
+https://docs.confluent.io/platform/current/installation/configuration/topic-configs.html#message-timestamp-after-max-ms[log.message.timestamp.after.max.ms].
++
+These properties limit the time difference between the message timestamp (from Logstash) and the Kafka broker receive time. 
+Messages can be rejected if the values are exceeded and `log.message.timestamp.type=CreateTime` is set. 
++
+These checks are ignored if `log.message.timestamp.type=LogAppendTime` is set. 
+
+* For Kafka version 0.10.0.0+ the message creation timestamp is set by Logstash and equals the initial timestamp of the event. 
+This behavior affects Kafka’s retention policy. 
+For example, if a Logstash event was created two weeks ago and Kafka retention is set to seven days, then when the message arrives in Kafka today it may be discarded immediately, because its timestamp is older than seven days.
++
+You can change this behavior by setting timestamps on message arrival instead.
+  The message is not discarded but kept for 7 more days. Set `log.message.timestamp.type` to `LogAppendTime` (default `CreateTime`) in your Kafka configuration.
+****
+
 [id="plugins-{type}s-{plugin}-aws_msk_iam_auth"]
 ==== AWS MSK IAM authentication
 If you use AWS MSK, the AWS MSK IAM access control enables you to handle both authentication and authorization for your MSK cluster with AWS IAM.

data/lib/logstash/inputs/kafka.rb

@@ -75,6 +75,8 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   #
   # * earliest: automatically reset the offset to the earliest offset
   # * latest: automatically reset the offset to the latest offset
+  # * by_duration:<duration>: automatically reset the offset to a configured duration from the current
+  #   timestamp (ISO 8601 format, e.g. by_duration:PT1H). Available since Apache Kafka 4.0.0.
   # * none: throw exception to the consumer if no previous offset is found for the consumer's group
   # * anything else: throw exception to the consumer.
   config :auto_offset_reset, :validate => :string

data/lib/logstash-integration-kafka_jars.rb

@@ -1,17 +1,17 @@
 # AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
 
 require 'jar_dependencies'
-require_jar('io.confluent', 'kafka-avro-serializer', '8.1.1')
-require_jar('org.apache.kafka', 'kafka-clients', '4.1.0')
-require_jar('at.yawk.lz4', 'lz4-java', '1.10.1')
-require_jar('io.confluent', 'kafka-schema-serializer', '8.1.1')
-require_jar('io.confluent', 'kafka-schema-registry-client', '8.1.1')
+require_jar('io.confluent', 'kafka-avro-serializer', '8.2.0')
+require_jar('org.apache.kafka', 'kafka-clients', '4.2.0')
+require_jar('io.confluent', 'kafka-schema-serializer', '8.2.0')
+require_jar('io.confluent', 'kafka-schema-registry-client', '8.2.0')
 require_jar('org.apache.avro', 'avro', '1.12.1')
 require_jar('org.apache.commons', 'commons-compress', '1.28.0')
 require_jar('com.google.guava', 'guava', '32.0.1-jre')
-require_jar('io.confluent', 'logredactor', '1.0.13')
-require_jar('io.confluent', 'common-utils', '8.1.1')
+require_jar('io.confluent', 'logredactor', '1.0.17')
+require_jar('io.confluent', 'common-utils', '8.2.0')
 require_jar('com.github.luben', 'zstd-jni', '1.5.6-10')
+require_jar('at.yawk.lz4', 'lz4-java', '1.10.1')
 require_jar('org.xerial.snappy', 'snappy-java', '1.1.10.7')
 require_jar('org.apache.httpcomponents.client5', 'httpclient5', '5.5')
 require_jar('org.slf4j', 'slf4j-api', '2.0.17')
@@ -28,7 +28,7 @@ require_jar('org.checkerframework', 'checker-qual', '3.33.0')
 require_jar('com.google.errorprone', 'error_prone_annotations', '2.18.0')
 require_jar('com.google.j2objc', 'j2objc-annotations', '2.8')
 require_jar('com.google.re2j', 're2j', '1.6')
-require_jar('io.confluent', 'logredactor-metrics', '1.0.13')
+require_jar('io.confluent', 'logredactor-metrics', '1.0.17')
 require_jar('com.eclipsesource.minimal-json', 'minimal-json', '0.9.5')
 require_jar('commons-codec', 'commons-codec', '1.19.0')
 require_jar('commons-io', 'commons-io', '2.20.0')

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '12.0.5'
+  s.version         = '12.1.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+

data/spec/fixtures/kafka_sasl_config.properties

@@ -0,0 +1,35 @@
+# SASL Configuration for integration tests
+listeners=PLAINTEXT://:9092,CONTROLLER://:9093,SASL_PLAINTEXT://:9094,SASL_SSL://:9095
+advertised.listeners=PLAINTEXT://localhost:9092,SASL_PLAINTEXT://localhost:9094,SASL_SSL://localhost:9095
+listener.security.protocol.map=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL
+
+# Enable multiple SASL mechanisms
+sasl.enabled.mechanisms=PLAIN,SCRAM-SHA-256,SCRAM-SHA-512
+sasl.mechanism.inter.broker.protocol=PLAIN
+
+# JAAS config for SASL_PLAINTEXT listener
+listener.name.sasl_plaintext.plain.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required \
+    username="admin" \
+    password="admin-secret" \
+    user_admin="admin-secret" \
+    user_logstash="logstash-secret";
+listener.name.sasl_plaintext.scram-sha-256.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required \
+    username="admin" \
+    password="admin-secret";
+listener.name.sasl_plaintext.scram-sha-512.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required \
+    username="admin" \
+    password="admin-secret";
+
+# JAAS config for SASL_SSL listener
+listener.name.sasl_ssl.plain.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required \
+    username="admin" \
+    password="admin-secret" \
+    user_admin="admin-secret" \
+    user_logstash="logstash-secret";
+listener.name.sasl_ssl.scram-sha-256.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required \
+    username="admin" \
+    password="admin-secret";
+listener.name.sasl_ssl.scram-sha-512.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required \
+    username="admin" \
+    password="admin-secret";
+

data/spec/integration/inputs/kafka_spec.rb

@@ -50,6 +50,18 @@ describe "inputs/kafka", :integration => true do
     { 'topics' => ['logstash_integration_topic_plain'], 'group_id' => group_id_5,
       'auto_offset_reset' => 'earliest', 'enable_auto_commit' => 'false' }
   end
+  let(:sasl_config) do
+    { 'topics' => ['logstash_integration_sasl_topic'], 'group_id' => group_id_6,
+      'bootstrap_servers' => 'localhost:9094',
+      'auto_offset_reset' => 'earliest',  'security_protocol' => 'SASL_PLAINTEXT' }
+  end
+  let(:sasl_ssl_config) do
+    { 'topics' => ['logstash_integration_sasl_topic'], 'group_id' => group_id_6,
+      'bootstrap_servers' => 'localhost:9095',
+      'auto_offset_reset' => 'earliest', 'security_protocol' => 'SASL_SSL',
+      'ssl_truststore_location' => File.join(Dir.pwd, 'tls_repository/clienttruststore.jks'),
+      'ssl_truststore_password' => 'changeit' }
+  end
   let(:timeout_seconds) { 30 }
   let(:num_events) { 103 }
 
@@ -264,53 +276,63 @@ describe "inputs/kafka", :integration => true do
     end
   end
 
-  # ToDo: add tests for other sasl config options as well (https://github.com/logstash-plugins/logstash-integration-kafka/issues/234)
-  context 'setting sasl_jaas_config' do
-    let(:base_config) do
-      {
-        'topics' => ['logstash_integration_topic_plain'],
-        'group_id' => rand(36**8).to_s(36),
-      }
+  context 'SASL authentication' do
+    shared_examples 'consumes all messages' do
+      it 'authenticates and consumes all messages' do
+        queue = consume_messages(consumer_config, timeout: timeout_seconds, event_count: num_events)
+        expect(queue.length).to eq(num_events)
+      end
     end
 
-    shared_examples 'sasl_jaas_config password handling' do
-      it 'stores sasl_jaas_config as password type' do
-        kafka_input = LogStash::Inputs::Kafka.new(consumer_config)
-        expect(kafka_input.sasl_jaas_config).to be_a(LogStash::Util::Password)
-        expect(kafka_input.sasl_jaas_config.value).to eq(jaas_config_value)
+    context 'over SASL_PLAINTEXT' do
+      context 'with PLAIN mechanism' do
+        let(:consumer_config) do
+          sasl_config.merge(
+            'sasl_mechanism' => 'PLAIN',
+            'sasl_jaas_config' => 'org.apache.kafka.common.security.plain.PlainLoginModule required username="logstash" password="logstash-secret";',
+          )
+        end
+
+        include_examples 'consumes all messages'
       end
 
-      it 'does not expose password in inspect output' do
-        kafka_input = LogStash::Inputs::Kafka.new(consumer_config)
-        expect(kafka_input.sasl_jaas_config.inspect).to eq('<password>')
-        expect(kafka_input.sasl_jaas_config.inspect).not_to include('admin-secret')
+      context 'with SCRAM-SHA-256 mechanism' do
+        let(:consumer_config) do
+          sasl_config.merge(
+            'sasl_mechanism' => 'SCRAM-SHA-256',
+            'sasl_jaas_config' => 'org.apache.kafka.common.security.scram.ScramLoginModule required username="logstash" password="logstash-secret";',
+          )
+        end
+
+        include_examples 'consumes all messages'
       end
-    end
 
-    context 'with single-line config' do
-      let(:jaas_config_value) { 'org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="admin-secret";' }
-      let(:consumer_config) { base_config.merge('sasl_jaas_config' => jaas_config_value) }
+      context 'with SCRAM-SHA-512 mechanism' do
+        let(:consumer_config) do
+          sasl_config.merge(
+            'sasl_mechanism' => 'SCRAM-SHA-512',
+            'sasl_jaas_config' => 'org.apache.kafka.common.security.scram.ScramLoginModule required username="logstash" password="logstash-secret";',
+          )
+        end
 
-      include_examples 'sasl_jaas_config password handling'
+        include_examples 'consumes all messages'
+      end
     end
 
-    context 'with multiline config' do
-      let(:jaas_config_value) do
-        <<~JAAS
-          org.apache.kafka.common.security.plain.PlainLoginModule required
-            username="admin"
-            password="admin-secret"
-            user_admin="admin-secret"
-            user_alice="alice-secret";
-        JAAS
-      end
-      let(:consumer_config) { base_config.merge('sasl_jaas_config' => jaas_config_value) }
+    context 'over SASL_SSL' do
+      context 'with PLAIN mechanism' do
+        let(:consumer_config) do
+          sasl_ssl_config.merge(
+            'sasl_mechanism' => 'PLAIN',
+            'sasl_jaas_config' => 'org.apache.kafka.common.security.plain.PlainLoginModule required username="logstash" password="logstash-secret";',
+          )
+        end
 
-      include_examples 'sasl_jaas_config password handling'
+        include_examples 'consumes all messages'
+      end
     end
   end
 
-
   context "static membership 'group.instance.id' setting" do
     let(:base_config) do
       {

data/spec/unit/inputs/kafka_spec.rb

@@ -414,6 +414,30 @@ describe LogStash::Inputs::Kafka do
     end
   end
 
+  context 'auto_offset_reset config' do
+    let(:base_props) do
+      {
+        'bootstrap.servers' => 'localhost:9092',
+        'key.deserializer' => 'org.apache.kafka.common.serialization.StringDeserializer',
+        'value.deserializer' => 'org.apache.kafka.common.serialization.StringDeserializer'
+      }
+    end
+
+    %w[earliest latest none by_duration:PT1H by_duration:P1D by_duration:PT30S by_duration:PT15M].each do |value|
+      it "accepts '#{value}' as a valid auto.offset.reset value" do
+        props = base_props.merge('auto.offset.reset' => value)
+        expect { org.apache.kafka.clients.consumer.ConsumerConfig.new(props) }.to_not raise_error
+      end
+    end
+
+    %w[by_duration: by_duration:INVALID by_duration:-PT1H by_duration:abc].each do |value|
+      it "rejects '#{value}' as an invalid auto.offset.reset value" do
+        props = base_props.merge('auto.offset.reset' => value)
+        expect { org.apache.kafka.clients.consumer.ConsumerConfig.new(props) }.to raise_error(org.apache.kafka.common.config.ConfigException)
+      end
+    end
+  end
+
   context 'string integer config' do
     let(:config) { super().merge('session_timeout_ms' => '25000',
                                  'max_poll_interval_ms' => '345000',

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-kafka
 version: !ruby/object:Gem::Version
-  version: 12.0.5
+  version: 12.1.0
 platform: java
 authors:
 - Elastic
 bindir: bin
 cert_chain: []
-date: 2026-03-11 00:00:00.000000000 Z
+date: 2026-03-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core-plugin-api
@@ -236,6 +236,7 @@ files:
 - spec/check_docs_spec.rb
 - spec/fixtures/jaas.config
 - spec/fixtures/jaas3.config
+- spec/fixtures/kafka_sasl_config.properties
 - spec/fixtures/pwd
 - spec/fixtures/trust-store_stub.jks
 - spec/integration/inputs/kafka_spec.rb
@@ -260,12 +261,12 @@ files:
 - vendor/jar-dependencies/com/google/re2j/re2j/1.6/re2j-1.6.jar
 - vendor/jar-dependencies/commons-codec/commons-codec/1.19.0/commons-codec-1.19.0.jar
 - vendor/jar-dependencies/commons-io/commons-io/2.20.0/commons-io-2.20.0.jar
-- vendor/jar-dependencies/io/confluent/common-utils/8.1.1/common-utils-8.1.1.jar
-- vendor/jar-dependencies/io/confluent/kafka-avro-serializer/8.1.1/kafka-avro-serializer-8.1.1.jar
-- vendor/jar-dependencies/io/confluent/kafka-schema-registry-client/8.1.1/kafka-schema-registry-client-8.1.1.jar
-- vendor/jar-dependencies/io/confluent/kafka-schema-serializer/8.1.1/kafka-schema-serializer-8.1.1.jar
-- vendor/jar-dependencies/io/confluent/logredactor-metrics/1.0.13/logredactor-metrics-1.0.13.jar
-- vendor/jar-dependencies/io/confluent/logredactor/1.0.13/logredactor-1.0.13.jar
+- vendor/jar-dependencies/io/confluent/common-utils/8.2.0/common-utils-8.2.0.jar
+- vendor/jar-dependencies/io/confluent/kafka-avro-serializer/8.2.0/kafka-avro-serializer-8.2.0.jar
+- vendor/jar-dependencies/io/confluent/kafka-schema-registry-client/8.2.0/kafka-schema-registry-client-8.2.0.jar
+- vendor/jar-dependencies/io/confluent/kafka-schema-serializer/8.2.0/kafka-schema-serializer-8.2.0.jar
+- vendor/jar-dependencies/io/confluent/logredactor-metrics/1.0.17/logredactor-metrics-1.0.17.jar
+- vendor/jar-dependencies/io/confluent/logredactor/1.0.17/logredactor-1.0.17.jar
 - vendor/jar-dependencies/io/swagger/core/v3/swagger-annotations/2.2.29/swagger-annotations-2.2.29.jar
 - vendor/jar-dependencies/org/apache/avro/avro/1.12.1/avro-1.12.1.jar
 - vendor/jar-dependencies/org/apache/commons/commons-compress/1.28.0/commons-compress-1.28.0.jar
@@ -273,7 +274,7 @@ files:
 - vendor/jar-dependencies/org/apache/httpcomponents/client5/httpclient5/5.5/httpclient5-5.5.jar
 - vendor/jar-dependencies/org/apache/httpcomponents/core5/httpcore5-h2/5.3.4/httpcore5-h2-5.3.4.jar
 - vendor/jar-dependencies/org/apache/httpcomponents/core5/httpcore5/5.3.4/httpcore5-5.3.4.jar
-- vendor/jar-dependencies/org/apache/kafka/kafka-clients/4.1.0/kafka-clients-4.1.0.jar
+- vendor/jar-dependencies/org/apache/kafka/kafka-clients/4.2.0/kafka-clients-4.2.0.jar
 - vendor/jar-dependencies/org/checkerframework/checker-qual/3.33.0/checker-qual-3.33.0.jar
 - vendor/jar-dependencies/org/slf4j/slf4j-api/2.0.17/slf4j-api-2.0.17.jar
 - vendor/jar-dependencies/org/xerial/snappy/snappy-java/1.1.10.7/snappy-java-1.1.10.7.jar
@@ -307,6 +308,7 @@ test_files:
 - spec/check_docs_spec.rb
 - spec/fixtures/jaas.config
 - spec/fixtures/jaas3.config
+- spec/fixtures/kafka_sasl_config.properties
 - spec/fixtures/pwd
 - spec/fixtures/trust-store_stub.jks
 - spec/integration/inputs/kafka_spec.rb