logstash-integration-kafka 11.6.4-java → 11.7.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 24a6a245c978681eeadad520b621e5f7a85bdbc21ba73917f3e70cf7a043c74f
-  data.tar.gz: 3ab1acf046db153809a1fee3ecc12129bf19545a2519e868bb2d90e7f394acaa
+  metadata.gz: 73e5319cf0e521282ced1615e3cb864ae58fd4195b39a67e42ff995cb22d5004
+  data.tar.gz: 380f7e07b99231ae047fc8545e7be56132f92b7d8179ae97a1f7f95c49a01033
 SHA512:
-  metadata.gz: 48acaae432ac06a78ecae3010c0b89285ae13315c1e2e08641c8b3d983bd1e4374f9563b0ec92b09763316c56cccbe6b80cf2d5aca6b0f95f60b9e35c4c1514e
-  data.tar.gz: 1f35ff308f3d30afde4215b4ccf68683ba71218ab78860d0addc37fe82c74b480dc5c2be423d0abbc89e8ad43264b7e8c29d7b331fdf3c51a17466eb5abff3fa
+  metadata.gz: 88d9dcbab467d2dc1af0e79e3b76af0cfdb3e2398644f1d4337414965c67813ee6d6829478c3bce8d082bbde269e34c134867a0831d0c4235cf9855c388597cf
+  data.tar.gz: 840eb2502efc99a1517790f315bb4a181697345eb422d36e64f7cf4d577999415acc43c32174ce905e31d0cd5cc00bd8f1b16962e26c78617db8fef3397280dd

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 11.7.0
+  - Add `reconnect_backoff_max_ms` option for configuring kafka client [#204](https://github.com/logstash-plugins/logstash-integration-kafka/pull/204)
+
 ## 11.6.4
   - Display exception chain comes from kafka client [#200](https://github.com/logstash-plugins/logstash-integration-kafka/pull/200)
 

data/docs/input-kafka.asciidoc

@@ -145,6 +145,7 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai
 | <<plugins-{type}s-{plugin}-poll_timeout_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-receive_buffer_bytes>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-reconnect_backoff_ms>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-reconnect_backoff_max_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-request_timeout_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-retry_backoff_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-sasl_client_callback_handler_class>> |<<string,string>>|No
@@ -561,6 +562,15 @@ The amount of time to wait before attempting to reconnect to a given host.
 This avoids repeatedly connecting to a host in a tight loop.
 This backoff applies to all requests sent by the consumer to the broker.
 
+[id="plugins-{type}s-{plugin}-reconnect_backoff_max_ms"]
+===== `reconnect_backoff_max_ms`
+
+* Value type is <<number,number>>
+* Default value is `1000` milliseconds.
+
+The maximum amount of time in milliseconds to wait when reconnecting to a broker that has repeatedly failed to connect.
+If provided, the backoff per host will increase exponentially for each consecutive connection failure, up to this maximum.
+
 [id="plugins-{type}s-{plugin}-request_timeout_ms"]
 ===== `request_timeout_ms` 
 

data/docs/output-kafka.asciidoc

@@ -115,6 +115,7 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai
 | <<plugins-{type}s-{plugin}-partitioner>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-receive_buffer_bytes>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-reconnect_backoff_ms>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-reconnect_backoff_max_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-request_timeout_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-retries>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-retry_backoff_ms>> |<<number,number>>|No
@@ -373,6 +374,15 @@ The size of the TCP receive buffer to use when reading data
 
 The amount of time to wait before attempting to reconnect to a given host when a connection fails.
 
+[id="plugins-{type}s-{plugin}-reconnect_backoff_max_ms"]
+===== `reconnect_backoff_max_ms`
+
+* Value type is <<number,number>>
+* Default value is `1000`.
+
+The maximum amount of time in milliseconds to wait when reconnecting to a broker that has repeatedly failed to connect.
+If provided, the backoff per host will increase exponentially for each consecutive connection failure, up to this maximum.
+
 [id="plugins-{type}s-{plugin}-request_timeout_ms"]
 ===== `request_timeout_ms` 
 

data/lib/logstash/inputs/kafka.rb

@@ -167,6 +167,9 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   # This avoids repeatedly connecting to a host in a tight loop.
   # This backoff applies to all connection attempts by the client to a broker.
   config :reconnect_backoff_ms, :validate => :number, :default => 50 # Kafka default
+  # The maximum amount of time in milliseconds to wait when reconnecting to a broker that has repeatedly failed to connect.
+  # If provided, the backoff per host will increase exponentially for each consecutive connection failure, up to this maximum.
+  config :reconnect_backoff_max_ms, :validate => :number, :default => 1000 # Kafka default
   # The amount of time to wait before attempting to retry a failed fetch request
   # to a given topic partition. This avoids repeated fetching-and-failing in a tight loop.
   config :retry_backoff_ms, :validate => :number, :default => 100 # Kafka default
@@ -457,6 +460,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
       props.put(kafka::PARTITION_ASSIGNMENT_STRATEGY_CONFIG, partition_assignment_strategy_class) unless partition_assignment_strategy.nil?
       props.put(kafka::RECEIVE_BUFFER_CONFIG, receive_buffer_bytes.to_s) unless receive_buffer_bytes.nil?
       props.put(kafka::RECONNECT_BACKOFF_MS_CONFIG, reconnect_backoff_ms.to_s) unless reconnect_backoff_ms.nil?
+      props.put(kafka::RECONNECT_BACKOFF_MAX_MS_CONFIG, reconnect_backoff_max_ms.to_s) unless reconnect_backoff_max_ms.nil?
       props.put(kafka::REQUEST_TIMEOUT_MS_CONFIG, request_timeout_ms.to_s) unless request_timeout_ms.nil?
       props.put(kafka::RETRY_BACKOFF_MS_CONFIG, retry_backoff_ms.to_s) unless retry_backoff_ms.nil?
       props.put(kafka::SEND_BUFFER_CONFIG, send_buffer_bytes.to_s) unless send_buffer_bytes.nil?

data/lib/logstash/outputs/kafka.rb

@@ -116,6 +116,9 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   config :receive_buffer_bytes, :validate => :number, :default => 32_768 # (32KB) Kafka default
   # The amount of time to wait before attempting to reconnect to a given host when a connection fails.
   config :reconnect_backoff_ms, :validate => :number, :default => 50 # Kafka default
+  # The maximum amount of time in milliseconds to wait when reconnecting to a broker that has repeatedly failed to connect.
+  # If provided, the backoff per host will increase exponentially for each consecutive connection failure, up to this maximum.
+  config :reconnect_backoff_max_ms, :validate => :number, :default => 1000 # Kafka default
   # The default retry behavior is to retry until successful. To prevent data loss,
   # the use of this setting is discouraged.
   #
@@ -372,6 +375,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
       end
       props.put(kafka::RECEIVE_BUFFER_CONFIG, receive_buffer_bytes.to_s) unless receive_buffer_bytes.nil?
       props.put(kafka::RECONNECT_BACKOFF_MS_CONFIG, reconnect_backoff_ms.to_s) unless reconnect_backoff_ms.nil?
+      props.put(kafka::RECONNECT_BACKOFF_MAX_MS_CONFIG, reconnect_backoff_max_ms.to_s) unless reconnect_backoff_max_ms.nil?
       props.put(kafka::REQUEST_TIMEOUT_MS_CONFIG, request_timeout_ms.to_s) unless request_timeout_ms.nil?
       props.put(kafka::RETRIES_CONFIG, retries.to_s) unless retries.nil?
       props.put(kafka::RETRY_BACKOFF_MS_CONFIG, retry_backoff_ms.to_s) 

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '11.6.4'
+  s.version         = '11.7.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+

data/spec/unit/inputs/kafka_spec.rb

@@ -378,11 +378,15 @@ describe LogStash::Inputs::Kafka do
   end
 
   context 'string integer config' do
-    let(:config) { super().merge('session_timeout_ms' => '25000', 'max_poll_interval_ms' => '345000') }
+    let(:config) { super().merge('session_timeout_ms' => '25000',
+                                 'max_poll_interval_ms' => '345000',
+                                 'reconnect_backoff_max_ms' => '1500') }
 
     it "sets integer values" do
       expect(org.apache.kafka.clients.consumer.KafkaConsumer).
-          to receive(:new).with(hash_including('session.timeout.ms' => '25000', 'max.poll.interval.ms' => '345000')).
+          to receive(:new).with(hash_including('session.timeout.ms' => '25000',
+                                               'max.poll.interval.ms' => '345000',
+                                               'reconnect.backoff.max.ms' => '1500')).
               and_return kafka_client = double('kafka-consumer')
 
       expect( subject.send(:create_consumer, 'sample_client-1', 'group_instance_id') ).to be kafka_client
@@ -390,11 +394,15 @@ describe LogStash::Inputs::Kafka do
   end
 
   context 'integer config' do
-    let(:config) { super().merge('session_timeout_ms' => 25200, 'max_poll_interval_ms' => 123_000) }
+    let(:config) { super().merge('session_timeout_ms' => 25200,
+                                 'max_poll_interval_ms' => 123_000,
+                                 'reconnect_backoff_max_ms' => 1500) }
 
     it "sets integer values" do
       expect(org.apache.kafka.clients.consumer.KafkaConsumer).
-          to receive(:new).with(hash_including('session.timeout.ms' => '25200', 'max.poll.interval.ms' => '123000')).
+          to receive(:new).with(hash_including('session.timeout.ms' => '25200',
+                                               'max.poll.interval.ms' => '123000',
+                                               'reconnect.backoff.max.ms' => '1500')).
               and_return kafka_client = double('kafka-consumer')
 
       expect( subject.send(:create_consumer, 'sample_client-2', 'group_instance_id') ).to be kafka_client

data/spec/unit/outputs/kafka_spec.rb

@@ -114,7 +114,7 @@ describe "outputs/kafka" do
   end
 
   context "when KafkaProducer#send() raises a non-retriable exception" do
-    let(:failcount) { (rand * 10).to_i }
+    let(:failcount) { 3 }
 
     let(:exception_classes) { [
         org.apache.kafka.common.errors.SerializationException,

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-kafka
 version: !ruby/object:Gem::Version
-  version: 11.6.4
+  version: 11.7.0
 platform: java
 authors:
 - Elastic
 bindir: bin
 cert_chain: []
-date: 2025-08-28 00:00:00.000000000 Z
+date: 2025-10-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core-plugin-api