logstash-integration-kafka 11.5.4-java → 11.6.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2aa0fa8dec3f2fd40ca01d9204dfbacb514635584e9dd36d2b6eb26df2920801
-  data.tar.gz: b887f71474c06efbdf342c512fa6a37c14d8500fabb5a6cdd9c5373c2d7b192d
+  metadata.gz: 9d342010bbdc6c64b7380374583eb1d33648c5cd9e8b948e6d40c9c62c120963
+  data.tar.gz: 3bcf9b167d4cb7cc6d2857d6f81dce26bf09120fed71a2a75d7a54137f6d4a0a
 SHA512:
-  metadata.gz: 29297b83b191648c25a9cab409c0dd7b6182e517e6583a40ae5462a9bb249a47a7ef815a83579ea14dbc5207d3773f7e502a8af484340d1a48db66e5ccfeaf76
-  data.tar.gz: e4b7b2334cb17197c11f5cabf140cc3e67e4d5d96203b11b5187411482a957236d292edda61868562823daf559bac89430c99d9a73c9ba4392f62d3830babd0f
+  metadata.gz: 1d0097a7d05bbc52065e1f32f6fc8f600052f6eeb75a782bfd270763d73df8150fa74fdadb705bc1eb6e448ed725420642cb6aaeb016bfae058718ec3b8fe0ea
+  data.tar.gz: 99487e99f6fbfdb52f2b103a2f9fa9504a67253f9591ef3650ccbbcc7eac5cd5ca5170c841e692b17ac9b02d17c356519d6e4a3b7caf3e1a24825e65e3b9aee5

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 11.6.0
+  - Support additional `oauth` and `sasl` configuration options for configuring kafka client [#189](https://github.com/logstash-plugins/logstash-integration-kafka/pull/189)
+
 ## 11.5.4
   - Update kafka client to 3.8.1 and transitive dependencies [#188](https://github.com/logstash-plugins/logstash-integration-kafka/pull/188)
   - Removed Jar Dependencies dependency [#187](https://github.com/logstash-plugins/logstash-integration-kafka/pull/187)

data/docs/input-kafka.asciidoc

@@ -131,6 +131,13 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai
 | <<plugins-{type}s-{plugin}-request_timeout_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-retry_backoff_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-sasl_client_callback_handler_class>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-sasl_oauthbearer_token_endpoint_url>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-sasl_oauthbearer_scope_claim_name>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-sasl_login_callback_handler_class>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-sasl_login_connect_timeout_ms>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-sasl_login_read_timeout_ms>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-sasl_login_retry_backoff_ms>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-sasl_login_retry_backoff_max_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-sasl_jaas_config>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-sasl_mechanism>> |<<string,string>>|No
@@ -556,13 +563,62 @@ retries are exhausted.
 The amount of time to wait before attempting to retry a failed fetch request
 to a given topic partition. This avoids repeated fetching-and-failing in a tight loop.
 
-[id="plugins-{type}s-{plugin}-sasl_client_callback_handler_class""]
+[id="plugins-{type}s-{plugin}-sasl_client_callback_handler_class"]
 ===== `sasl_client_callback_handler_class`
-* Value type is <<string,string>>
-* There is no default value for this setting.
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
 
 The SASL client callback handler class the specified SASL mechanism should use.
 
+[id="plugins-{type}s-{plugin}-sasl_oauthbearer_token_endpoint_url"]
+===== `sasl_oauthbearer_token_endpoint_url`
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The URL for the OAuth 2.0 issuer token endpoint.
+
+[id="plugins-{type}s-{plugin}-sasl_oauthbearer_scope_claim_name"]
+===== `sasl_oauthbearer_scope_claim_name`
+  * Value type is <<string,string>>
+  * Default value is `"scope"`
+
+(optional) The override name of the scope claim.
+
+[id="plugins-{type}s-{plugin}-sasl_login_callback_handler_class"]
+===== `sasl_login_callback_handler_class`
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The SASL login callback handler class the specified SASL mechanism should use.
+
+[id="plugins-{type}s-{plugin}-sasl_login_connect_timeout_ms"]
+===== `sasl_login_connect_timeout_ms`
+  * Value type is <<number,number>>
+  * There is no default value for this setting.
+
+(optional) The duration, in milliseconds, for HTTPS connect timeout
+
+[id="plugins-{type}s-{plugin}-sasl_login_read_timeout_ms"]
+===== `sasl_login_read_timeout_ms`
+  * Value type is <<number,number>>
+  * There is no default value for this setting.
+
+(optional) The duration, in milliseconds, for HTTPS read timeout.
+
+[id="plugins-{type}s-{plugin}-sasl_login_retry_backoff_ms"]
+===== `sasl_login_retry_backoff_ms`
+  * Value type is <<number,number>>
+  * Default value is `100` milliseconds.
+
+(optional) The duration, in milliseconds, to wait between HTTPS call attempts.
+
+[id="plugins-{type}s-{plugin}-sasl_login_retry_backoff_max_ms"]
+===== `sasl_login_retry_backoff_max_ms`
+  * Value type is <<number,number>>
+  * Default value is `10000` milliseconds.
+
+(optional) The maximum duration, in milliseconds, for HTTPS call attempts.
+
 [id="plugins-{type}s-{plugin}-sasl_jaas_config"]
 ===== `sasl_jaas_config` 
 

data/docs/output-kafka.asciidoc

@@ -102,6 +102,13 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai
 | <<plugins-{type}s-{plugin}-retries>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-retry_backoff_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-sasl_client_callback_handler_class>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-sasl_oauthbearer_token_endpoint_url>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-sasl_oauthbearer_scope_claim_name>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-sasl_login_callback_handler_class>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-sasl_login_connect_timeout_ms>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-sasl_login_read_timeout_ms>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-sasl_login_retry_backoff_ms>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-sasl_login_retry_backoff_max_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-sasl_jaas_config>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-sasl_mechanism>> |<<string,string>>|No
@@ -392,13 +399,62 @@ In versions prior to 10.5.0, any exception is retried indefinitely unless the `r
 
 The amount of time to wait before attempting to retry a failed produce request to a given topic partition.
 
-[id="plugins-{type}s-{plugin}-sasl_client_callback_handler_class""]
+[id="plugins-{type}s-{plugin}-sasl_client_callback_handler_class"]
 ===== `sasl_client_callback_handler_class`
-* Value type is <<string,string>>
-* There is no default value for this setting.
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
 
 The SASL client callback handler class the specified SASL mechanism should use.
 
+[id="plugins-{type}s-{plugin}-sasl_oauthbearer_token_endpoint_url"]
+===== `sasl_oauthbearer_token_endpoint_url`
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The URL for the OAuth 2.0 issuer token endpoint.
+
+[id="plugins-{type}s-{plugin}-sasl_oauthbearer_scope_claim_name"]
+===== `sasl_oauthbearer_scope_claim_name`
+  * Value type is <<string,string>>
+  * Default value is `"scope"`
+
+(optional) The override name of the scope claim.
+
+[id="plugins-{type}s-{plugin}-sasl_login_callback_handler_class"]
+===== `sasl_login_callback_handler_class`
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The SASL login callback handler class the specified SASL mechanism should use.
+
+[id="plugins-{type}s-{plugin}-sasl_login_connect_timeout_ms"]
+===== `sasl_login_connect_timeout_ms`
+  * Value type is <<number,number>>
+  * There is no default value for this setting.
+
+(optional) The duration, in milliseconds, for HTTPS connect timeout
+
+[id="plugins-{type}s-{plugin}-sasl_login_read_timeout_ms"]
+===== `sasl_login_read_timeout_ms`
+  * Value type is <<number,number>>
+  * There is no default value for this setting.
+
+(optional) The duration, in milliseconds, for HTTPS read timeout.
+
+[id="plugins-{type}s-{plugin}-sasl_login_retry_backoff_ms"]
+===== `sasl_login_retry_backoff_ms`
+  * Value type is <<number,number>>
+  * Default value is `100` milliseconds.
+
+(optional) The duration, in milliseconds, to wait between HTTPS call attempts.
+
+[id="plugins-{type}s-{plugin}-sasl_login_retry_backoff_max_ms"]
+===== `sasl_login_retry_backoff_max_ms`
+  * Value type is <<number,number>>
+  * Default value is `10000` milliseconds.
+
+(optional) The maximum duration, in milliseconds, for HTTPS call attempts.
+
 [id="plugins-{type}s-{plugin}-sasl_jaas_config"]
 ===== `sasl_jaas_config` 
 

data/lib/logstash/inputs/kafka.rb

@@ -210,6 +210,20 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   config :security_protocol, :validate => ["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"], :default => "PLAINTEXT"
   # SASL client callback handler class
   config :sasl_client_callback_handler_class, :validate => :string
+  # The URL for the OAuth 2.0 issuer token endpoint.
+  config :sasl_oauthbearer_token_endpoint_url, :validate => :string
+  # (optional) The override name of the scope claim.
+  config :sasl_oauthbearer_scope_claim_name, :validate => :string, :default => 'scope' # Kafka default
+  # SASL login callback handler class
+  config :sasl_login_callback_handler_class, :validate => :string
+  # (optional) The duration, in milliseconds, for HTTPS connect timeout
+  config :sasl_login_connect_timeout_ms, :validate => :number
+  # (optional) The duration, in milliseconds, for HTTPS read timeout.
+  config :sasl_login_read_timeout_ms, :validate => :number
+  # (optional) The duration, in milliseconds, to wait between HTTPS call attempts.
+  config :sasl_login_retry_backoff_ms, :validate => :number, :default => 100 # Kafka default
+  # (optional) The maximum duration, in milliseconds, for HTTPS call attempts.
+  config :sasl_login_retry_backoff_max_ms, :validate => :number, :default => 10000 # Kafka default
   # http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. 
   # This may be any mechanism for which a security provider is available.
   # GSSAPI is the default mechanism.

data/lib/logstash/outputs/kafka.rb

@@ -149,6 +149,20 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   config :security_protocol, :validate => ["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"], :default => "PLAINTEXT"
   # SASL client callback handler class
   config :sasl_client_callback_handler_class, :validate => :string
+  # The URL for the OAuth 2.0 issuer token endpoint.
+  config :sasl_oauthbearer_token_endpoint_url, :validate => :string
+  # (optional) The override name of the scope claim.
+  config :sasl_oauthbearer_scope_claim_name, :validate => :string, :default => 'scope' # Kafka default
+  # SASL login callback handler class
+  config :sasl_login_callback_handler_class, :validate => :string
+  # (optional) The duration, in milliseconds, for HTTPS connect timeout
+  config :sasl_login_connect_timeout_ms, :validate => :number
+  # (optional) The duration, in milliseconds, for HTTPS read timeout.
+  config :sasl_login_read_timeout_ms, :validate => :number
+  # (optional) The duration, in milliseconds, to wait between HTTPS call attempts.
+  config :sasl_login_retry_backoff_ms, :validate => :number, :default => 100 # Kafka default
+  # (optional) The maximum duration, in milliseconds, for HTTPS call attempts.
+  config :sasl_login_retry_backoff_max_ms, :validate => :number, :default => 10000 # Kafka default
   # http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. 
   # This may be any mechanism for which a security provider is available.
   # GSSAPI is the default mechanism.

data/lib/logstash/plugin_mixins/kafka/common.rb

@@ -42,6 +42,13 @@ module LogStash module PluginMixins module Kafka
       props.put("sasl.kerberos.service.name", sasl_kerberos_service_name) unless sasl_kerberos_service_name.nil?
       props.put("sasl.jaas.config", sasl_jaas_config) unless sasl_jaas_config.nil?
       props.put("sasl.client.callback.handler.class", sasl_client_callback_handler_class) unless sasl_client_callback_handler_class.nil?
+      props.put("sasl.oauthbearer.token.endpoint.url", sasl_oauthbearer_token_endpoint_url) unless sasl_oauthbearer_token_endpoint_url.nil?
+      props.put("sasl.oauthbearer.scope.claim.name", sasl_oauthbearer_scope_claim_name) unless sasl_oauthbearer_scope_claim_name.nil?
+      props.put("sasl.login.callback.handler.class", sasl_login_callback_handler_class) unless sasl_login_callback_handler_class.nil?
+      props.put("sasl.login.connect.timeout.ms", sasl_login_connect_timeout_ms.to_s) unless sasl_login_connect_timeout_ms.nil?
+      props.put("sasl.login.read.timeout.ms", sasl_login_read_timeout_ms.to_s) unless sasl_login_read_timeout_ms.nil?
+      props.put("sasl.login.retry.backoff.ms", sasl_login_retry_backoff_ms.to_s) unless sasl_login_retry_backoff_ms.nil?
+      props.put("sasl.login.retry.backoff.max.ms", sasl_login_retry_backoff_max_ms.to_s) unless sasl_login_retry_backoff_max_ms.nil?
     end
 
     def reassign_dns_lookup

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '11.5.4'
+  s.version         = '11.6.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+

data/spec/unit/inputs/kafka_spec.rb

@@ -218,6 +218,54 @@ describe LogStash::Inputs::Kafka do
 
   end
 
+  context 'when oauth is configured' do
+    let(:config) { super().merge(
+      'security_protocol' => 'SASL_PLAINTEXT',
+      'sasl_mechanism' => 'OAUTHBEARER',
+      'sasl_oauthbearer_token_endpoint_url' => 'https://auth.example.com/token',
+      'sasl_oauthbearer_scope_claim_name' => 'custom_scope'
+    )}
+
+    it "sets oauth properties" do
+      expect(org.apache.kafka.clients.consumer.KafkaConsumer).
+        to receive(:new).with(hash_including(
+          'security.protocol' => 'SASL_PLAINTEXT',
+          'sasl.mechanism' => 'OAUTHBEARER',
+          'sasl.oauthbearer.token.endpoint.url' => 'https://auth.example.com/token',
+          'sasl.oauthbearer.scope.claim.name' => 'custom_scope'
+        )).and_return(kafka_client = double('kafka-consumer'))
+
+      expect(subject.send(:create_consumer, 'test-client-1', 'group_instance_id')).to be kafka_client
+    end
+  end
+
+  context 'when sasl is configured' do
+    let(:config) { super().merge(
+      'security_protocol' => 'SASL_PLAINTEXT',
+      'sasl_mechanism' => 'OAUTHBEARER',
+      'sasl_login_connect_timeout_ms' => 15000,
+      'sasl_login_read_timeout_ms' => 5000,
+      'sasl_login_retry_backoff_ms' => 200,
+      'sasl_login_retry_backoff_max_ms' => 15000,
+      'sasl_login_callback_handler_class' => 'org.example.CustomLoginHandler'
+    )}
+
+    it "sets sasl login properties" do
+      expect(org.apache.kafka.clients.consumer.KafkaConsumer).
+        to receive(:new).with(hash_including(
+          'security.protocol' => 'SASL_PLAINTEXT',
+          'sasl.mechanism' => 'OAUTHBEARER',
+          'sasl.login.connect.timeout.ms' => '15000',
+          'sasl.login.read.timeout.ms' => '5000',
+          'sasl.login.retry.backoff.ms' => '200',
+          'sasl.login.retry.backoff.max.ms' => '15000',
+          'sasl.login.callback.handler.class' => 'org.example.CustomLoginHandler'
+        )).and_return(kafka_client = double('kafka-consumer'))
+
+      expect(subject.send(:create_consumer, 'test-client-2', 'group_instance_id')).to be kafka_client
+    end
+  end
+
   describe "schema registry" do
     let(:base_config) do {
           'schema_registry_url' => 'http://localhost:8081',

data/spec/unit/outputs/kafka_spec.rb

@@ -9,6 +9,7 @@ describe "outputs/kafka" do
                                       '@timestamp' => LogStash::Timestamp.now}) }
 
   let(:future) { double('kafka producer future') }
+  subject { LogStash::Outputs::Kafka.new(config) }
 
   context 'when initializing' do
     it "should register" do
@@ -267,8 +268,6 @@ describe "outputs/kafka" do
       File.join(File.dirname(__FILE__), '../../fixtures/trust-store_stub.jks')
     end
 
-    subject { LogStash::Outputs::Kafka.new(config) }
-
     it 'sets empty ssl.endpoint.identification.algorithm' do
       expect(org.apache.kafka.clients.producer.KafkaProducer).
           to receive(:new).with(hash_including('ssl.endpoint.identification.algorithm' => ''))
@@ -283,4 +282,53 @@ describe "outputs/kafka" do
 
   end
 
+  context 'when oauth is configured' do
+    let(:config) {
+      simple_kafka_config.merge(
+        'security_protocol' => 'SASL_PLAINTEXT',
+        'sasl_mechanism' => 'OAUTHBEARER',
+        'sasl_oauthbearer_token_endpoint_url' => 'https://auth.example.com/token',
+        'sasl_oauthbearer_scope_claim_name' => 'custom_scope'
+      )
+    }
+
+    it "sets oauth properties" do
+      expect(org.apache.kafka.clients.producer.KafkaProducer).
+        to receive(:new).with(hash_including(
+          'security.protocol' => 'SASL_PLAINTEXT',
+          'sasl.mechanism' => 'OAUTHBEARER',
+          'sasl.oauthbearer.token.endpoint.url' => 'https://auth.example.com/token',
+          'sasl.oauthbearer.scope.claim.name' => 'custom_scope'
+        ))
+      subject.register
+    end
+  end
+
+  context 'when sasl is configured' do
+    let(:config) {
+      simple_kafka_config.merge(
+        'security_protocol' => 'SASL_PLAINTEXT',
+        'sasl_mechanism' => 'OAUTHBEARER',
+        'sasl_login_connect_timeout_ms' => 15000,
+        'sasl_login_read_timeout_ms' => 5000,
+        'sasl_login_retry_backoff_ms' => 200,
+        'sasl_login_retry_backoff_max_ms' => 15000,
+        'sasl_login_callback_handler_class' => 'org.example.CustomLoginHandler'
+      )
+    }
+
+    it "sets sasl login properties" do
+      expect(org.apache.kafka.clients.producer.KafkaProducer).
+        to receive(:new).with(hash_including(
+          'security.protocol' => 'SASL_PLAINTEXT',
+          'sasl.mechanism' => 'OAUTHBEARER',
+          'sasl.login.connect.timeout.ms' => '15000',
+          'sasl.login.read.timeout.ms' => '5000',
+          'sasl.login.retry.backoff.ms' => '200',
+          'sasl.login.retry.backoff.max.ms' => '15000',
+          'sasl.login.callback.handler.class' => 'org.example.CustomLoginHandler'
+        ))
+      subject.register
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-kafka
 version: !ruby/object:Gem::Version
-  version: 11.5.4
+  version: 11.6.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-12-18 00:00:00.000000000 Z
+date: 2025-01-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement