logstash-integration-kafka 11.6.4-java → 12.0.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 24a6a245c978681eeadad520b621e5f7a85bdbc21ba73917f3e70cf7a043c74f
-  data.tar.gz: 3ab1acf046db153809a1fee3ecc12129bf19545a2519e868bb2d90e7f394acaa
+  metadata.gz: 744abb784463992c54a691cf16cbab0eca25f9b5cc73a54d9808f15bca223021
+  data.tar.gz: d35f68e589ee17adb865b7edcb1c9627b1345e9de93c5037828c15e8e83defdb
 SHA512:
-  metadata.gz: 48acaae432ac06a78ecae3010c0b89285ae13315c1e2e08641c8b3d983bd1e4374f9563b0ec92b09763316c56cccbe6b80cf2d5aca6b0f95f60b9e35c4c1514e
-  data.tar.gz: 1f35ff308f3d30afde4215b4ccf68683ba71218ab78860d0addc37fe82c74b480dc5c2be423d0abbc89e8ad43264b7e8c29d7b331fdf3c51a17466eb5abff3fa
+  metadata.gz: b85e870af8ae8dc030320ecd0c9ad312d72cceacb64b2e0ac313ca3629b3a1c24a3029f1512485b79ccd68e272eccddfee1ac32a314dc562af7cc1762b82aa53
+  data.tar.gz: 89fe6266982b250eb000650c91d8fdfc653cfc02d9cdf570c743edd5d7290ffd1735b0fbc3b68b018fb65215b67a909cdc5bb1dfb05e025256b35de0cca6b1af

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## 12.0.0
+  - Update kafka client to 4.1.0 and transitive dependencies [#205](https://github.com/logstash-plugins/logstash-integration-kafka/pull/205)
+    - Breaking Change: partitioner options `default` and `uniform_sticky` are removed
+    - `linger_ms` default value changed from 0 to 5
+  - Add `group_protocols` options for configuring Kafka consumer rebalance protocol
+    - Setting `group_protocols => consumer` opts in to the new consumer group protocol
+
+## 11.7.0
+  - Add `reconnect_backoff_max_ms` option for configuring kafka client [#204](https://github.com/logstash-plugins/logstash-integration-kafka/pull/204)
+
 ## 11.6.4
   - Display exception chain comes from kafka client [#200](https://github.com/logstash-plugins/logstash-integration-kafka/pull/200)
 

data/docs/index.asciidoc

@@ -1,7 +1,7 @@
 :plugin: kafka
 :type: integration
 :no_codec:
-:kafka_client: 3.9.1
+:kafka_client: 4.1.0
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!

data/docs/input-kafka.asciidoc

@@ -2,8 +2,8 @@
 :plugin: kafka
 :type: input
 :default_codec: plain
-:kafka_client: 3.9.1
-:kafka_client_doc: 39
+:kafka_client: 4.1.0
+:kafka_client_doc: 41
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!
@@ -132,6 +132,7 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai
 | <<plugins-{type}s-{plugin}-fetch_min_bytes>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-group_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-group_instance_id>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-group_protocol>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-heartbeat_interval_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-isolation_level>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-jaas_path>> |a valid filesystem path|No
@@ -145,6 +146,7 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai
 | <<plugins-{type}s-{plugin}-poll_timeout_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-receive_buffer_bytes>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-reconnect_backoff_ms>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-reconnect_backoff_max_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-request_timeout_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-retry_backoff_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-sasl_client_callback_handler_class>> |<<string,string>>|No
@@ -411,6 +413,21 @@ You can set this value to use information such as a hostname, an IP, or anything
 NOTE: In cases when multiple threads are configured and `consumer_threads` is greater than one, a suffix is appended to
 the `group_instance_id` to avoid collisions.
 
+[id="plugins-{type}s-{plugin}-group_protocol"]
+===== `group_protocol`
+
+* Value can be either of: `classic`, `consumer`
+* Default value is `classic`.
+
+Specifies the consumer group rebalance protocol used by the Kafka client.
+
+`classic` is the default protocol. During a rebalance, all consumer instances pause message processing until partition assignments are complete.
+
+`consumer` is an incremental rebalance protocol introduced in Kafka 4. It avoids global synchronization barriers by only pausing partitions that are reassigned.
+When using `consumer`, the following settings **cannot be configured**:
+`partition_assignment_strategy`, `heartbeat_interval_ms`, and `session_timeout_ms`.
+
+
 [id="plugins-{type}s-{plugin}-heartbeat_interval_ms"]
 ===== `heartbeat_interval_ms` 
 
@@ -561,6 +578,15 @@ The amount of time to wait before attempting to reconnect to a given host.
 This avoids repeatedly connecting to a host in a tight loop.
 This backoff applies to all requests sent by the consumer to the broker.
 
+[id="plugins-{type}s-{plugin}-reconnect_backoff_max_ms"]
+===== `reconnect_backoff_max_ms`
+
+* Value type is <<number,number>>
+* Default value is `1000` milliseconds.
+
+The maximum amount of time in milliseconds to wait when reconnecting to a broker that has repeatedly failed to connect.
+If provided, the backoff per host will increase exponentially for each consecutive connection failure, up to this maximum.
+
 [id="plugins-{type}s-{plugin}-request_timeout_ms"]
 ===== `request_timeout_ms` 
 

data/docs/output-kafka.asciidoc

@@ -2,8 +2,8 @@
 :plugin: kafka
 :type: output
 :default_codec: plain
-:kafka_client: 3.9.1
-:kafka_client_doc: 39
+:kafka_client: 4.1.0
+:kafka_client_doc: 41
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!
@@ -115,6 +115,7 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai
 | <<plugins-{type}s-{plugin}-partitioner>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-receive_buffer_bytes>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-reconnect_backoff_ms>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-reconnect_backoff_max_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-request_timeout_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-retries>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-retry_backoff_ms>> |<<number,number>>|No
@@ -284,7 +285,7 @@ Serializer class for the key of the message
 ===== `linger_ms` 
 
   * Value type is <<number,number>>
-  * Default value is `0`
+  * Default value is `5`
 
 The producer groups together any records that arrive in between request
 transmissions into a single batched request. Normally this occurs only under
@@ -348,14 +349,11 @@ The max time in milliseconds before a metadata refresh is forced.
 * Value type is <<string,string>>
 * There is no default value for this setting.
 
-The default behavior is to hash the `message_key` of an event to get the partition.
-When no message key is present, the plugin picks a partition in a round-robin fashion.
+By not setting this value, the plugin uses the built-in partitioning strategy provided by the Kafka client. Read more about the "partitioner.class" on the Kafka documentation.
 
-Available options for choosing a partitioning strategy are as follows:
+Available option is as follows:
 
-* `default` use the default partitioner as described above
 * `round_robin` distributes writes to all partitions equally, regardless of `message_key`
-* `uniform_sticky` sticks to a partition for the duration of a batch than randomly picks a new one
 
 [id="plugins-{type}s-{plugin}-receive_buffer_bytes"]
 ===== `receive_buffer_bytes` 
@@ -373,6 +371,15 @@ The size of the TCP receive buffer to use when reading data
 
 The amount of time to wait before attempting to reconnect to a given host when a connection fails.
 
+[id="plugins-{type}s-{plugin}-reconnect_backoff_max_ms"]
+===== `reconnect_backoff_max_ms`
+
+* Value type is <<number,number>>
+* Default value is `1000`.
+
+The maximum amount of time in milliseconds to wait when reconnecting to a broker that has repeatedly failed to connect.
+If provided, the backoff per host will increase exponentially for each consecutive connection failure, up to this maximum.
+
 [id="plugins-{type}s-{plugin}-request_timeout_ms"]
 ===== `request_timeout_ms` 
 

data/lib/logstash/inputs/kafka.rb

@@ -129,6 +129,12 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   # consumer crated by each thread an artificial suffix is appended to the user provided `group_instance_id`
   # to avoid clashing.
   config :group_instance_id, :validate => :string
+  # `classic` is the "stop-the-world" rebalances.
+  #   Any consumer restart or failure triggers a full-group rebalance, pausing processing for all consumers.
+  # `consumer` is an incremental rebalance protocol that avoids global sync barriers,
+  #   pausing only the partitions that are reassigned.
+  #   It cannot set along with `partition_assignment_strategy`, `heartbeat_interval_ms` and `session_timeout_ms`
+  config :group_protocol, :validate => ["classic", "consumer"], :default => "classic"
   # The expected time between heartbeats to the consumer coordinator. Heartbeats are used to ensure 
   # that the consumer's session stays active and to facilitate rebalancing when new
   # consumers join or leave the group. The value must be set lower than
@@ -167,6 +173,9 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   # This avoids repeatedly connecting to a host in a tight loop.
   # This backoff applies to all connection attempts by the client to a broker.
   config :reconnect_backoff_ms, :validate => :number, :default => 50 # Kafka default
+  # The maximum amount of time in milliseconds to wait when reconnecting to a broker that has repeatedly failed to connect.
+  # If provided, the backoff per host will increase exponentially for each consecutive connection failure, up to this maximum.
+  config :reconnect_backoff_max_ms, :validate => :number, :default => 1000 # Kafka default
   # The amount of time to wait before attempting to retry a failed fetch request
   # to a given topic partition. This avoids repeated fetching-and-failing in a tight loop.
   config :retry_backoff_ms, :validate => :number, :default => 100 # Kafka default
@@ -290,6 +299,8 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
     reassign_dns_lookup
     @pattern ||= java.util.regex.Pattern.compile(@topics_pattern) unless @topics_pattern.nil?
     check_schema_registry_parameters
+
+    set_group_protocol!
   end
 
   METADATA_NONE     = Set[].freeze
@@ -447,6 +458,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
       props.put(kafka::FETCH_MIN_BYTES_CONFIG, fetch_min_bytes.to_s) unless fetch_min_bytes.nil?
       props.put(kafka::GROUP_ID_CONFIG, group_id)
       props.put(kafka::GROUP_INSTANCE_ID_CONFIG, group_instance_id) unless group_instance_id.nil?
+      props.put(kafka::GROUP_PROTOCOL_CONFIG, group_protocol)
       props.put(kafka::HEARTBEAT_INTERVAL_MS_CONFIG, heartbeat_interval_ms.to_s) unless heartbeat_interval_ms.nil?
       props.put(kafka::ISOLATION_LEVEL_CONFIG, isolation_level)
       props.put(kafka::KEY_DESERIALIZER_CLASS_CONFIG, key_deserializer_class)
@@ -457,6 +469,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
       props.put(kafka::PARTITION_ASSIGNMENT_STRATEGY_CONFIG, partition_assignment_strategy_class) unless partition_assignment_strategy.nil?
       props.put(kafka::RECEIVE_BUFFER_CONFIG, receive_buffer_bytes.to_s) unless receive_buffer_bytes.nil?
       props.put(kafka::RECONNECT_BACKOFF_MS_CONFIG, reconnect_backoff_ms.to_s) unless reconnect_backoff_ms.nil?
+      props.put(kafka::RECONNECT_BACKOFF_MAX_MS_CONFIG, reconnect_backoff_max_ms.to_s) unless reconnect_backoff_max_ms.nil?
       props.put(kafka::REQUEST_TIMEOUT_MS_CONFIG, request_timeout_ms.to_s) unless request_timeout_ms.nil?
       props.put(kafka::RETRY_BACKOFF_MS_CONFIG, retry_backoff_ms.to_s) unless retry_backoff_ms.nil?
       props.put(kafka::SEND_BUFFER_CONFIG, send_buffer_bytes.to_s) unless send_buffer_bytes.nil?
@@ -467,7 +480,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
       props.put("security.protocol", security_protocol) unless security_protocol.nil?
       if schema_registry_url
         props.put(kafka::VALUE_DESERIALIZER_CLASS_CONFIG, Java::io.confluent.kafka.serializers.KafkaAvroDeserializer.java_class)
-        serdes_config = Java::io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig
+        serdes_config = Java::io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig
         props.put(serdes_config::SCHEMA_REGISTRY_URL_CONFIG, schema_registry_url.uri.to_s)
         if schema_registry_proxy && !schema_registry_proxy.empty?
           props.put(serdes_config::PROXY_HOST, @schema_registry_proxy_host)
@@ -509,6 +522,25 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
     end
   end
 
+  # In order to use group_protocol => consumer, heartbeat_interval_ms, session_timeout_ms and partition_assignment_strategy need to be unset
+  # If any of these are not using the default value of the plugin, we raise a configuration error
+  def set_group_protocol!
+    return unless group_protocol == "consumer"
+
+    heartbeat_overridden = heartbeat_interval_ms != self.class.get_config.dig("heartbeat_interval_ms", :default)
+    session_overridden   = session_timeout_ms != self.class.get_config.dig("session_timeout_ms", :default)
+    strategy_defined     = !partition_assignment_strategy.nil?
+
+    if strategy_defined || heartbeat_overridden || session_overridden
+      raise LogStash::ConfigurationError, "group_protocol cannot be set to 'consumer' "\
+        "when any of partition_assignment_strategy, heartbeat_interval_ms or session_timeout_ms is set"
+    end
+
+    @heartbeat_interval_ms = nil
+    @session_timeout_ms = nil
+    logger.debug("Settings 'heartbeat_interval_ms' and 'session_timeout_ms' have been reset since 'group_protocol' is configured as 'consumer'")
+  end
+
   def partition_assignment_strategy_class
     case partition_assignment_strategy
     when 'range'

data/lib/logstash/outputs/kafka.rb

@@ -101,7 +101,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   # This setting accomplishes this by adding a small amount of artificial delay—that is,
   # rather than immediately sending out a record the producer will wait for up to the given delay
   # to allow other records to be sent so that the sends can be batched together.
-  config :linger_ms, :validate => :number, :default => 0 # Kafka default
+  config :linger_ms, :validate => :number, :default => 5 # Kafka default
   # The maximum size of a request
   config :max_request_size, :validate => :number, :default => 1_048_576 # (1MB) Kafka default
   # The key for the message
@@ -110,12 +110,15 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   config :message_headers, :validate => :hash, :default => {}
   # the timeout setting for initial metadata request to fetch topic metadata.
   config :metadata_fetch_timeout_ms, :validate => :number, :default => 60_000
-  # Partitioner to use - can be `default`, `uniform_sticky`, `round_robin` or a fully qualified class name of a custom partitioner.
+  # Partitioner to use - can be `round_robin` or a fully qualified class name of a custom partitioner.
   config :partitioner, :validate => :string
   # The size of the TCP receive buffer to use when reading data
   config :receive_buffer_bytes, :validate => :number, :default => 32_768 # (32KB) Kafka default
   # The amount of time to wait before attempting to reconnect to a given host when a connection fails.
   config :reconnect_backoff_ms, :validate => :number, :default => 50 # Kafka default
+  # The maximum amount of time in milliseconds to wait when reconnecting to a broker that has repeatedly failed to connect.
+  # If provided, the backoff per host will increase exponentially for each consecutive connection failure, up to this maximum.
+  config :reconnect_backoff_max_ms, :validate => :number, :default => 1000 # Kafka default
   # The default retry behavior is to retry until successful. To prevent data loss,
   # the use of this setting is discouraged.
   #
@@ -366,12 +369,13 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
       props.put(kafka::LINGER_MS_CONFIG, linger_ms.to_s)
       props.put(kafka::MAX_REQUEST_SIZE_CONFIG, max_request_size.to_s)
       props.put(kafka::METADATA_MAX_AGE_CONFIG, metadata_max_age_ms.to_s) unless metadata_max_age_ms.nil?
-      unless partitioner.nil?
-        props.put(kafka::PARTITIONER_CLASS_CONFIG, partitioner = partitioner_class)
+      partitioner_class&.tap do |partitioner|
+        props.put(kafka::PARTITIONER_CLASS_CONFIG, partitioner)
         logger.debug('producer configured using partitioner', :partitioner_class => partitioner)
       end
       props.put(kafka::RECEIVE_BUFFER_CONFIG, receive_buffer_bytes.to_s) unless receive_buffer_bytes.nil?
       props.put(kafka::RECONNECT_BACKOFF_MS_CONFIG, reconnect_backoff_ms.to_s) unless reconnect_backoff_ms.nil?
+      props.put(kafka::RECONNECT_BACKOFF_MAX_MS_CONFIG, reconnect_backoff_max_ms.to_s) unless reconnect_backoff_max_ms.nil?
       props.put(kafka::REQUEST_TIMEOUT_MS_CONFIG, request_timeout_ms.to_s) unless request_timeout_ms.nil?
       props.put(kafka::RETRIES_CONFIG, retries.to_s) unless retries.nil?
       props.put(kafka::RETRY_BACKOFF_MS_CONFIG, retry_backoff_ms.to_s) 
@@ -401,19 +405,12 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   end
 
   def partitioner_class
-    case partitioner
-    when 'round_robin'
-      'org.apache.kafka.clients.producer.RoundRobinPartitioner'
-    when 'uniform_sticky'
-      'org.apache.kafka.clients.producer.UniformStickyPartitioner'
-    when 'default'
-      'org.apache.kafka.clients.producer.internals.DefaultPartitioner'
-    else
-      unless partitioner.index('.')
-        raise LogStash::ConfigurationError, "unsupported partitioner: #{partitioner.inspect}"
-      end
-      partitioner # assume a fully qualified class-name
-    end
+    return nil if partitioner.nil?
+    return 'org.apache.kafka.clients.producer.RoundRobinPartitioner' if partitioner == 'round_robin'
+
+    raise LogStash::ConfigurationError, "unsupported partitioner: #{partitioner.inspect}" unless partitioner.include?('.')
+
+    partitioner
   end
 
 end #class LogStash::Outputs::Kafka

data/lib/logstash-integration-kafka_jars.rb

@@ -1,12 +1,13 @@
 # AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
 
 require 'jar_dependencies'
-require_jar('io.confluent', 'kafka-avro-serializer', '7.9.1')
-require_jar('io.confluent', 'kafka-schema-serializer', '7.9.1')
+require_jar('io.confluent', 'kafka-avro-serializer', '8.0.0')
+require_jar('io.confluent', 'kafka-schema-serializer', '8.0.0')
 require_jar('org.apache.avro', 'avro', '1.11.4')
-require_jar('io.confluent', 'kafka-schema-registry-client', '7.9.1')
-require_jar('org.apache.kafka', 'kafka-clients', '3.9.1')
+require_jar('io.confluent', 'kafka-schema-registry-client', '8.0.0')
+require_jar('com.fasterxml.jackson.datatype', 'jackson-datatype-jdk8', '2.16.0')
+require_jar('org.apache.kafka', 'kafka-clients', '4.1.0')
 require_jar('org.slf4j', 'slf4j-api', '1.7.36')
-require_jar('com.github.luben', 'zstd-jni', '1.5.6-8')
+require_jar('com.github.luben', 'zstd-jni', '1.5.6-10')
 require_jar('org.lz4', 'lz4-java', '1.8.0')
 require_jar('org.xerial.snappy', 'snappy-java', '1.1.10.7')

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '11.6.4'
+  s.version         = '12.0.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+

data/spec/fixtures/jaas.config

@@ -1,5 +1,5 @@
 SchemaRegistry-Props {
-  org.eclipse.jetty.jaas.spi.PropertyFileLoginModule required
+  org.eclipse.jetty.security.jaas.spi.PropertyFileLoginModule required
   file="build/confluent_platform/etc/schema-registry/pwd"
   debug="true";
 };

data/spec/fixtures/jaas3.config

@@ -0,0 +1,5 @@
+SchemaRegistry-Props {
+  org.eclipse.jetty.jaas.spi.PropertyFileLoginModule required
+  file="build/confluent_platform/etc/schema-registry/pwd"
+  debug="true";
+};

data/spec/integration/inputs/kafka_spec.rb

@@ -187,6 +187,83 @@ describe "inputs/kafka", :integration => true do
     end
   end
 
+  context 'setting group_protocol' do
+    let(:test_topic) { 'logstash_integration_partitioner_topic' }
+    let(:consumer_config) do
+      plain_config.merge(
+        "topics" => [test_topic],
+        'group_protocol' => group_protocol,
+        "partition_assignment_strategy" => partition_assignment_strategy,
+        "heartbeat_interval_ms" => heartbeat_interval_ms,
+        "session_timeout_ms" => session_timeout_ms
+      )
+    end
+    let(:group_protocol) { nil }
+    let(:partition_assignment_strategy) { nil }
+    let(:heartbeat_interval_ms) { LogStash::Inputs::Kafka.get_config().dig("heartbeat_interval_ms", :default) }
+    let(:session_timeout_ms) { LogStash::Inputs::Kafka.get_config().dig("session_timeout_ms", :default) }
+
+    describe "group_protocol = classic" do
+      let(:group_protocol) { 'classic' }
+
+      it 'passes register check' do
+        kafka_input = LogStash::Inputs::Kafka.new(consumer_config)
+        expect {
+          kafka_input.register
+        }.to_not raise_error
+
+        expect( kafka_input.instance_variable_get(:@heartbeat_interval_ms)).eql?(heartbeat_interval_ms)
+        expect( kafka_input.instance_variable_get(:@session_timeout_ms)).eql?(session_timeout_ms)
+      end
+    end
+
+    describe "group_protocol = consumer" do
+      let(:group_protocol) { 'consumer' }
+
+      describe "passes register check with supported config" do
+        it 'reset unsupported config to nil' do
+          kafka_input = LogStash::Inputs::Kafka.new(consumer_config)
+          expect {
+            kafka_input.register
+          }.to_not raise_error
+
+          expect( kafka_input.instance_variable_get(:@heartbeat_interval_ms)).to be_nil
+          expect( kafka_input.instance_variable_get(:@session_timeout_ms)).to be_nil
+        end
+      end
+
+      {
+        partition_assignment_strategy: 'range',
+        heartbeat_interval_ms: 2000,
+        session_timeout_ms: 6000
+      }.each do |config_key, config_value|
+        context "with unsupported config #{config_key}" do
+          let(config_key) { config_value }
+
+          it 'raises LogStash::ConfigurationError' do
+            kafka_input = LogStash::Inputs::Kafka.new(consumer_config)
+            expect {
+              kafka_input.register
+            }.to raise_error(LogStash::ConfigurationError, /group_protocol cannot be set to.*consumer.*/)
+          end
+        end
+      end
+
+      context "with valid config" do
+        let(:test_topic) { 'logstash_integration_topic_plain' }
+        let(:manual_commit_config) do
+          consumer_config.merge(
+            'enable_auto_commit' => 'false'
+          )
+        end
+        it 'consume data' do
+          queue = consume_messages(manual_commit_config, timeout: timeout_seconds, event_count: num_events)
+          expect(queue.length).to eq(num_events)
+        end
+      end
+    end
+  end
+
   context "static membership 'group.instance.id' setting" do
     let(:base_config) do
       {
@@ -486,7 +563,7 @@ describe "Deserializing with the schema registry", :integration => true do
     def delete_topic_if_exists(topic_name, user = nil, password = nil)
       props = java.util.Properties.new
       props.put(Java::org.apache.kafka.clients.admin.AdminClientConfig::BOOTSTRAP_SERVERS_CONFIG, "localhost:9092")
-      serdes_config = Java::io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig
+      serdes_config = Java::io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig
       unless user.nil?
         props.put(serdes_config::BASIC_AUTH_CREDENTIALS_SOURCE, 'USER_INFO')
         props.put(serdes_config::USER_INFO_CONFIG,  "#{user}:#{password}")
@@ -495,7 +572,7 @@ describe "Deserializing with the schema registry", :integration => true do
       topics_list = admin_client.listTopics().names().get()
       if topics_list.contains(topic_name)
         result = admin_client.deleteTopics([topic_name])
-        result.values.get(topic_name).get()
+        result.topicNameValues().get(topic_name).get()
       end
     end
 
@@ -503,7 +580,7 @@ describe "Deserializing with the schema registry", :integration => true do
       props = java.util.Properties.new
       config = org.apache.kafka.clients.producer.ProducerConfig
 
-      serdes_config = Java::io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig
+      serdes_config = Java::io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig
       props.put(serdes_config::SCHEMA_REGISTRY_URL_CONFIG, "http://localhost:8081")
 
       props.put(config::BOOTSTRAP_SERVERS_CONFIG, "localhost:9092")

data/spec/integration/outputs/kafka_spec.rb

@@ -164,7 +164,7 @@ describe "outputs/kafka", :integration => true do
     let(:test_topic) { 'logstash_integration_topic3' }
 
     before :each do
-      config = base_config.merge("topic_id" => test_topic, "partitioner" => 'org.apache.kafka.clients.producer.UniformStickyPartitioner')
+      config = base_config.merge("topic_id" => test_topic, "partitioner" => 'org.apache.kafka.clients.producer.RoundRobinPartitioner')
       load_kafka_data(config) do # let's have a bit more (diverse) dataset
         num_events.times.collect do
           LogStash::Event.new.tap do |e|
@@ -212,7 +212,7 @@ describe "outputs/kafka", :integration => true do
 
   context 'setting partitioner' do
     let(:test_topic) { 'logstash_integration_partitioner_topic' }
-    let(:partitioner) { nil }
+    let(:partitioner) { 'round_robin' }
 
     before :each do
       @messages_offset = fetch_messages_from_all_partitions
@@ -221,13 +221,8 @@ describe "outputs/kafka", :integration => true do
       load_kafka_data(config)
     end
 
-    [ 'default', 'round_robin', 'uniform_sticky' ].each do |partitioner|
-      describe partitioner do
-        let(:partitioner) { partitioner }
-        it 'loads data' do
-          expect(fetch_messages_from_all_partitions - @messages_offset).to eql num_events
-        end
-      end
+    it 'loads data' do
+      expect(fetch_messages_from_all_partitions - @messages_offset).to eql num_events
     end
 
     def fetch_messages_from_all_partitions

data/spec/unit/inputs/kafka_spec.rb

@@ -378,11 +378,15 @@ describe LogStash::Inputs::Kafka do
   end
 
   context 'string integer config' do
-    let(:config) { super().merge('session_timeout_ms' => '25000', 'max_poll_interval_ms' => '345000') }
+    let(:config) { super().merge('session_timeout_ms' => '25000',
+                                 'max_poll_interval_ms' => '345000',
+                                 'reconnect_backoff_max_ms' => '1500') }
 
     it "sets integer values" do
       expect(org.apache.kafka.clients.consumer.KafkaConsumer).
-          to receive(:new).with(hash_including('session.timeout.ms' => '25000', 'max.poll.interval.ms' => '345000')).
+          to receive(:new).with(hash_including('session.timeout.ms' => '25000',
+                                               'max.poll.interval.ms' => '345000',
+                                               'reconnect.backoff.max.ms' => '1500')).
               and_return kafka_client = double('kafka-consumer')
 
       expect( subject.send(:create_consumer, 'sample_client-1', 'group_instance_id') ).to be kafka_client
@@ -390,11 +394,15 @@ describe LogStash::Inputs::Kafka do
   end
 
   context 'integer config' do
-    let(:config) { super().merge('session_timeout_ms' => 25200, 'max_poll_interval_ms' => 123_000) }
+    let(:config) { super().merge('session_timeout_ms' => 25200,
+                                 'max_poll_interval_ms' => 123_000,
+                                 'reconnect_backoff_max_ms' => 1500) }
 
     it "sets integer values" do
       expect(org.apache.kafka.clients.consumer.KafkaConsumer).
-          to receive(:new).with(hash_including('session.timeout.ms' => '25200', 'max.poll.interval.ms' => '123000')).
+          to receive(:new).with(hash_including('session.timeout.ms' => '25200',
+                                               'max.poll.interval.ms' => '123000',
+                                               'reconnect.backoff.max.ms' => '1500')).
               and_return kafka_client = double('kafka-consumer')
 
       expect( subject.send(:create_consumer, 'sample_client-2', 'group_instance_id') ).to be kafka_client

data/spec/unit/outputs/kafka_spec.rb

@@ -114,7 +114,7 @@ describe "outputs/kafka" do
   end
 
   context "when KafkaProducer#send() raises a non-retriable exception" do
-    let(:failcount) { (rand * 10).to_i }
+    let(:failcount) { 3 }
 
     let(:exception_classes) { [
         org.apache.kafka.common.errors.SerializationException,

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-kafka
 version: !ruby/object:Gem::Version
-  version: 11.6.4
+  version: 12.0.0
 platform: java
 authors:
 - Elastic
 bindir: bin
 cert_chain: []
-date: 2025-08-28 00:00:00.000000000 Z
+date: 2025-10-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core-plugin-api
@@ -235,6 +235,7 @@ files:
 - logstash-integration-kafka.gemspec
 - spec/check_docs_spec.rb
 - spec/fixtures/jaas.config
+- spec/fixtures/jaas3.config
 - spec/fixtures/pwd
 - spec/fixtures/trust-store_stub.jks
 - spec/integration/inputs/kafka_spec.rb
@@ -242,12 +243,13 @@ files:
 - spec/unit/inputs/avro_schema_fixture_payment.asvc
 - spec/unit/inputs/kafka_spec.rb
 - spec/unit/outputs/kafka_spec.rb
-- vendor/jar-dependencies/com/github/luben/zstd-jni/1.5.6-8/zstd-jni-1.5.6-8.jar
-- vendor/jar-dependencies/io/confluent/kafka-avro-serializer/7.9.1/kafka-avro-serializer-7.9.1.jar
-- vendor/jar-dependencies/io/confluent/kafka-schema-registry-client/7.9.1/kafka-schema-registry-client-7.9.1.jar
-- vendor/jar-dependencies/io/confluent/kafka-schema-serializer/7.9.1/kafka-schema-serializer-7.9.1.jar
+- vendor/jar-dependencies/com/fasterxml/jackson/datatype/jackson-datatype-jdk8/2.16.0/jackson-datatype-jdk8-2.16.0.jar
+- vendor/jar-dependencies/com/github/luben/zstd-jni/1.5.6-10/zstd-jni-1.5.6-10.jar
+- vendor/jar-dependencies/io/confluent/kafka-avro-serializer/8.0.0/kafka-avro-serializer-8.0.0.jar
+- vendor/jar-dependencies/io/confluent/kafka-schema-registry-client/8.0.0/kafka-schema-registry-client-8.0.0.jar
+- vendor/jar-dependencies/io/confluent/kafka-schema-serializer/8.0.0/kafka-schema-serializer-8.0.0.jar
 - vendor/jar-dependencies/org/apache/avro/avro/1.11.4/avro-1.11.4.jar
-- vendor/jar-dependencies/org/apache/kafka/kafka-clients/3.9.1/kafka-clients-3.9.1.jar
+- vendor/jar-dependencies/org/apache/kafka/kafka-clients/4.1.0/kafka-clients-4.1.0.jar
 - vendor/jar-dependencies/org/lz4/lz4-java/1.8.0/lz4-java-1.8.0.jar
 - vendor/jar-dependencies/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar
 - vendor/jar-dependencies/org/xerial/snappy/snappy-java/1.1.10.7/snappy-java-1.1.10.7.jar
@@ -279,6 +281,7 @@ summary: Integration with Kafka - input and output plugins
 test_files:
 - spec/check_docs_spec.rb
 - spec/fixtures/jaas.config
+- spec/fixtures/jaas3.config
 - spec/fixtures/pwd
 - spec/fixtures/trust-store_stub.jks
 - spec/integration/inputs/kafka_spec.rb