logstash-integration-kafka 11.6.3-java → 11.7.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6438f327044c91c39fb03b639116ba47bb4b035aa2e47d01404485a94a1ac44c
-  data.tar.gz: c338ad50a044ebecb2ef9c3f478ed6012ef74503da73f4a5d144fa7c7ed83533
+  metadata.gz: 73e5319cf0e521282ced1615e3cb864ae58fd4195b39a67e42ff995cb22d5004
+  data.tar.gz: 380f7e07b99231ae047fc8545e7be56132f92b7d8179ae97a1f7f95c49a01033
 SHA512:
-  metadata.gz: 88e409679b56af0ed1ea3cfdc05fd8f16f0e582a5f7087e139d4c54ad85c7d4f772ab5ed02d47b1c0b82ad95c49e5e162a7cced7b0d264737a79fd58670af627
-  data.tar.gz: 92aa81a8834428fdb512dccd3780a9bb3568cde4c266f301a0db6c8d9fbf21b1449d5321e878e8a96955acc8f9236c4bbf25e35a92639f99f105bbe6967f9bf3
+  metadata.gz: 88d9dcbab467d2dc1af0e79e3b76af0cfdb3e2398644f1d4337414965c67813ee6d6829478c3bce8d082bbde269e34c134867a0831d0c4235cf9855c388597cf
+  data.tar.gz: 840eb2502efc99a1517790f315bb4a181697345eb422d36e64f7cf4d577999415acc43c32174ce905e31d0cd5cc00bd8f1b16962e26c78617db8fef3397280dd

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 11.7.0
+  - Add `reconnect_backoff_max_ms` option for configuring kafka client [#204](https://github.com/logstash-plugins/logstash-integration-kafka/pull/204)
+
+## 11.6.4
+  - Display exception chain comes from kafka client [#200](https://github.com/logstash-plugins/logstash-integration-kafka/pull/200)
+
 ## 11.6.3
   - Update kafka client to 3.9.1 and transitive dependencies [#193](https://github.com/logstash-plugins/logstash-integration-kafka/pull/193)
 

data/docs/input-kafka.asciidoc

@@ -145,6 +145,7 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai
 | <<plugins-{type}s-{plugin}-poll_timeout_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-receive_buffer_bytes>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-reconnect_backoff_ms>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-reconnect_backoff_max_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-request_timeout_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-retry_backoff_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-sasl_client_callback_handler_class>> |<<string,string>>|No
@@ -561,6 +562,15 @@ The amount of time to wait before attempting to reconnect to a given host.
 This avoids repeatedly connecting to a host in a tight loop.
 This backoff applies to all requests sent by the consumer to the broker.
 
+[id="plugins-{type}s-{plugin}-reconnect_backoff_max_ms"]
+===== `reconnect_backoff_max_ms`
+
+* Value type is <<number,number>>
+* Default value is `1000` milliseconds.
+
+The maximum amount of time in milliseconds to wait when reconnecting to a broker that has repeatedly failed to connect.
+If provided, the backoff per host will increase exponentially for each consecutive connection failure, up to this maximum.
+
 [id="plugins-{type}s-{plugin}-request_timeout_ms"]
 ===== `request_timeout_ms` 
 

data/docs/output-kafka.asciidoc

@@ -115,6 +115,7 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai
 | <<plugins-{type}s-{plugin}-partitioner>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-receive_buffer_bytes>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-reconnect_backoff_ms>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-reconnect_backoff_max_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-request_timeout_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-retries>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-retry_backoff_ms>> |<<number,number>>|No
@@ -373,6 +374,15 @@ The size of the TCP receive buffer to use when reading data
 
 The amount of time to wait before attempting to reconnect to a given host when a connection fails.
 
+[id="plugins-{type}s-{plugin}-reconnect_backoff_max_ms"]
+===== `reconnect_backoff_max_ms`
+
+* Value type is <<number,number>>
+* Default value is `1000`.
+
+The maximum amount of time in milliseconds to wait when reconnecting to a broker that has repeatedly failed to connect.
+If provided, the backoff per host will increase exponentially for each consecutive connection failure, up to this maximum.
+
 [id="plugins-{type}s-{plugin}-request_timeout_ms"]
 ===== `request_timeout_ms` 
 

data/lib/logstash/inputs/kafka.rb

@@ -167,6 +167,9 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   # This avoids repeatedly connecting to a host in a tight loop.
   # This backoff applies to all connection attempts by the client to a broker.
   config :reconnect_backoff_ms, :validate => :number, :default => 50 # Kafka default
+  # The maximum amount of time in milliseconds to wait when reconnecting to a broker that has repeatedly failed to connect.
+  # If provided, the backoff per host will increase exponentially for each consecutive connection failure, up to this maximum.
+  config :reconnect_backoff_max_ms, :validate => :number, :default => 1000 # Kafka default
   # The amount of time to wait before attempting to retry a failed fetch request
   # to a given topic partition. This avoids repeated fetching-and-failing in a tight loop.
   config :retry_backoff_ms, :validate => :number, :default => 100 # Kafka default
@@ -457,6 +460,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
       props.put(kafka::PARTITION_ASSIGNMENT_STRATEGY_CONFIG, partition_assignment_strategy_class) unless partition_assignment_strategy.nil?
       props.put(kafka::RECEIVE_BUFFER_CONFIG, receive_buffer_bytes.to_s) unless receive_buffer_bytes.nil?
       props.put(kafka::RECONNECT_BACKOFF_MS_CONFIG, reconnect_backoff_ms.to_s) unless reconnect_backoff_ms.nil?
+      props.put(kafka::RECONNECT_BACKOFF_MAX_MS_CONFIG, reconnect_backoff_max_ms.to_s) unless reconnect_backoff_max_ms.nil?
       props.put(kafka::REQUEST_TIMEOUT_MS_CONFIG, request_timeout_ms.to_s) unless request_timeout_ms.nil?
       props.put(kafka::RETRY_BACKOFF_MS_CONFIG, retry_backoff_ms.to_s) unless retry_backoff_ms.nil?
       props.put(kafka::SEND_BUFFER_CONFIG, send_buffer_bytes.to_s) unless send_buffer_bytes.nil?

data/lib/logstash/outputs/kafka.rb

@@ -116,6 +116,9 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   config :receive_buffer_bytes, :validate => :number, :default => 32_768 # (32KB) Kafka default
   # The amount of time to wait before attempting to reconnect to a given host when a connection fails.
   config :reconnect_backoff_ms, :validate => :number, :default => 50 # Kafka default
+  # The maximum amount of time in milliseconds to wait when reconnecting to a broker that has repeatedly failed to connect.
+  # If provided, the backoff per host will increase exponentially for each consecutive connection failure, up to this maximum.
+  config :reconnect_backoff_max_ms, :validate => :number, :default => 1000 # Kafka default
   # The default retry behavior is to retry until successful. To prevent data loss,
   # the use of this setting is discouraged.
   #
@@ -372,6 +375,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
       end
       props.put(kafka::RECEIVE_BUFFER_CONFIG, receive_buffer_bytes.to_s) unless receive_buffer_bytes.nil?
       props.put(kafka::RECONNECT_BACKOFF_MS_CONFIG, reconnect_backoff_ms.to_s) unless reconnect_backoff_ms.nil?
+      props.put(kafka::RECONNECT_BACKOFF_MAX_MS_CONFIG, reconnect_backoff_max_ms.to_s) unless reconnect_backoff_max_ms.nil?
       props.put(kafka::REQUEST_TIMEOUT_MS_CONFIG, request_timeout_ms.to_s) unless request_timeout_ms.nil?
       props.put(kafka::RETRIES_CONFIG, retries.to_s) unless retries.nil?
       props.put(kafka::RETRY_BACKOFF_MS_CONFIG, retry_backoff_ms.to_s) 
@@ -391,9 +395,11 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
 
       org.apache.kafka.clients.producer.KafkaProducer.new(props)
     rescue => e
-      logger.error("Unable to create Kafka producer from given configuration",
-                   :kafka_error_message => e,
-                   :cause => e.respond_to?(:getCause) ? e.getCause() : nil)
+      cause_error = e
+      while cause_error
+        logger.error("Kafka producer error chain", :kafka_error_message => "#{cause_error.class}: #{cause_error.message}")
+        cause_error = cause_error.respond_to?(:getCause) ? cause_error.getCause() : nil
+      end
       raise e
     end
   end

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '11.6.3'
+  s.version         = '11.7.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+

data/spec/unit/inputs/kafka_spec.rb

@@ -378,11 +378,15 @@ describe LogStash::Inputs::Kafka do
   end
 
   context 'string integer config' do
-    let(:config) { super().merge('session_timeout_ms' => '25000', 'max_poll_interval_ms' => '345000') }
+    let(:config) { super().merge('session_timeout_ms' => '25000',
+                                 'max_poll_interval_ms' => '345000',
+                                 'reconnect_backoff_max_ms' => '1500') }
 
     it "sets integer values" do
       expect(org.apache.kafka.clients.consumer.KafkaConsumer).
-          to receive(:new).with(hash_including('session.timeout.ms' => '25000', 'max.poll.interval.ms' => '345000')).
+          to receive(:new).with(hash_including('session.timeout.ms' => '25000',
+                                               'max.poll.interval.ms' => '345000',
+                                               'reconnect.backoff.max.ms' => '1500')).
               and_return kafka_client = double('kafka-consumer')
 
       expect( subject.send(:create_consumer, 'sample_client-1', 'group_instance_id') ).to be kafka_client
@@ -390,11 +394,15 @@ describe LogStash::Inputs::Kafka do
   end
 
   context 'integer config' do
-    let(:config) { super().merge('session_timeout_ms' => 25200, 'max_poll_interval_ms' => 123_000) }
+    let(:config) { super().merge('session_timeout_ms' => 25200,
+                                 'max_poll_interval_ms' => 123_000,
+                                 'reconnect_backoff_max_ms' => 1500) }
 
     it "sets integer values" do
       expect(org.apache.kafka.clients.consumer.KafkaConsumer).
-          to receive(:new).with(hash_including('session.timeout.ms' => '25200', 'max.poll.interval.ms' => '123000')).
+          to receive(:new).with(hash_including('session.timeout.ms' => '25200',
+                                               'max.poll.interval.ms' => '123000',
+                                               'reconnect.backoff.max.ms' => '1500')).
               and_return kafka_client = double('kafka-consumer')
 
       expect( subject.send(:create_consumer, 'sample_client-2', 'group_instance_id') ).to be kafka_client

data/spec/unit/outputs/kafka_spec.rb

@@ -114,7 +114,7 @@ describe "outputs/kafka" do
   end
 
   context "when KafkaProducer#send() raises a non-retriable exception" do
-    let(:failcount) { (rand * 10).to_i }
+    let(:failcount) { 3 }
 
     let(:exception_classes) { [
         org.apache.kafka.common.errors.SerializationException,

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-kafka
 version: !ruby/object:Gem::Version
-  version: 11.6.3
+  version: 11.7.0
 platform: java
 authors:
 - Elastic
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-06-12 00:00:00.000000000 Z
+date: 2025-10-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -19,7 +19,6 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
-  name: logstash-core-plugin-api
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -31,12 +30,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.99'
 - !ruby/object:Gem::Dependency
+  name: logstash-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 8.3.0
-  name: logstash-core
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -45,12 +44,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 8.3.0
 - !ruby/object:Gem::Dependency
+  name: logstash-codec-json
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-json
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -59,12 +58,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-plain
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -73,6 +72,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: stud
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -81,7 +81,6 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 0.1.0
-  name: stud
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -93,6 +92,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.1.0
 - !ruby/object:Gem::Dependency
+  name: manticore
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -101,7 +101,6 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 1.0.0
-  name: manticore
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -113,12 +112,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 1.0.0
 - !ruby/object:Gem::Dependency
+  name: logstash-mixin-deprecation_logger_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: logstash-mixin-deprecation_logger_support
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -127,12 +126,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
+  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -141,12 +140,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: logstash-codec-line
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-line
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -155,12 +154,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: rspec-wait
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: rspec-wait
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -169,12 +168,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: digest-crc
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.5.1
-  name: digest-crc
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -183,12 +182,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.5.1
 - !ruby/object:Gem::Dependency
+  name: ruby-kafka
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: ruby-kafka
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -197,12 +196,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: snappy
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: snappy
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -259,7 +258,6 @@ metadata:
   logstash_plugin: 'true'
   logstash_group: integration
   integration_plugins: logstash-input-kafka,logstash-output-kafka
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -275,8 +273,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
-signing_key:
+rubygems_version: 3.6.3
 specification_version: 4
 summary: Integration with Kafka - input and output plugins
 test_files: