logstash-integration-kafka 10.7.6-java → 10.8.2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fc9c66beb8b3a0074d0fa5090dc8896eedd796428e4b7644d8d96e159c7bed2f
-  data.tar.gz: 780f7aeac690ee7a23e954e94c2359d8e9527bcd1464dde007ce7e55c4a023f8
+  metadata.gz: 2e11499f791fd2bd71102ddb6f8e52fcbd9791feecf4107c7ebd27dc9334cc34
+  data.tar.gz: 779cd381a6e53155f61457229df4fc827ceda02ef4e069d4eab0240dafa4ab83
 SHA512:
-  metadata.gz: db2e1a6eefc076887de8b42944fe5e0d4ddf025915faf55a4f150710772b08c170e0b4bd629871ca89f4b45fa36b572eb15191c284dec5ad62409c228343e9c5
-  data.tar.gz: b1ae9060e3cf83f6b7c6ca8d949ab2f78696b7a983c7154d8db01e2004875ece63f202ac5778095112adf9597b2d54cedac0ed79682b59bb0bd1b9002265fd36
+  metadata.gz: 5577048fe1adee0c28b012c0bf6eb0ff594b7be4760f7b7ebdf9072473fb0a1806e24997dd7b0659c21f205396a4d7eccb10bc4e789541c35fb7e32610562632
+  data.tar.gz: 7ee0e7428d63957a57e927a3f0408bcdddf94efad944c5cf0acbac3c1031be55b6751ad0290ea134e5201f10f9e26f2804c6d68150e925f651c9db27847bab5e

data/CHANGELOG.md

@@ -1,3 +1,19 @@
+## 10.8.2
+  - [DOC] Updates description of `enable_auto_commit=false` to clarify that the commit happens after data is fetched AND written to the queue [#90](https://github.com/logstash-plugins/logstash-integration-kafka/pull/90)
+  - Fix: update to Gradle 7 [#104](https://github.com/logstash-plugins/logstash-integration-kafka/pull/104)
+  - [DOC] Clarify Kafka client does not support proxy [#103](https://github.com/logstash-plugins/logstash-integration-kafka/pull/103)
+
+## 10.8.1
+  - [DOC] Removed a setting recommendation that is no longer applicable for Kafka 2.0+ [#99](https://github.com/logstash-plugins/logstash-integration-kafka/pull/99)
+
+## 10.8.0
+  - Added config setting to enable schema registry validation to be skipped when an authentication scheme unsupported
+    by the validator is used [#97](https://github.com/logstash-plugins/logstash-integration-kafka/pull/97)
+
+## 10.7.7
+  - Fix: Correct the settings to allow basic auth to work properly, either by setting `schema_registry_key/secret` or embedding username/password in the
+    url [#94](https://github.com/logstash-plugins/logstash-integration-kafka/pull/94)
+
 ## 10.7.6
   - Test: specify development dependency version [#91](https://github.com/logstash-plugins/logstash-integration-kafka/pull/91)
 

data/docs/input-kafka.asciidoc

@@ -42,6 +42,13 @@ This input supports connecting to Kafka over:
 
 By default security is disabled but can be turned on as needed.
 
+[NOTE]
+=======
+This plugin does not support using a proxy when communicating to the Kafka broker.  
+
+This plugin does support using a proxy when communicating to the Schema Registry using the <<plugins-{type}s-{plugin}-schema_registry_proxy>> option.
+=======
+
 The Logstash Kafka consumer handles group management and uses the default offset management
 strategy using Kafka topics.
 
@@ -128,6 +135,7 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai
 | <<plugins-{type}s-{plugin}-schema_registry_proxy>> |<<uri,uri>>|No
 | <<plugins-{type}s-{plugin}-schema_registry_secret>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-schema_registry_url>> |<<uri,uri>>|No
+| <<plugins-{type}s-{plugin}-schema_registry_validation>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-security_protocol>> |<<string,string>>, one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No
 | <<plugins-{type}s-{plugin}-send_buffer_bytes>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-session_timeout_ms>> |<<number,number>>|No
@@ -275,7 +283,7 @@ which the consumption will begin.
 
 If true, periodically commit to Kafka the offsets of messages already returned by
 the consumer. If value is `false` however, the offset is committed every time the
-consumer fetches the data from the topic.
+consumer writes data fetched from the topic to the in-memory or persistent queue.
 
 [id="plugins-{type}s-{plugin}-exclude_internal_topics"]
 ===== `exclude_internal_topics` 
@@ -414,7 +422,6 @@ The maximum delay between invocations of poll() when using consumer group manage
 an upper bound on the amount of time that the consumer can be idle before fetching more records. 
 If poll() is not called before expiration of this timeout, then the consumer is considered failed and 
 the group will rebalance in order to reassign the partitions to another member.
-The value of the configuration `request_timeout_ms` must always be larger than `max_poll_interval_ms`. ???
 
 [id="plugins-{type}s-{plugin}-max_poll_records"]
 ===== `max_poll_records` 
@@ -576,6 +583,18 @@ The schemas must follow a naming convention with the pattern <topic name>-value.
 Use either the Schema Registry config option or the
 <<plugins-{type}s-{plugin}-value_deserializer_class>> config option, but not both.
 
+[id="plugins-{type}s-{plugin}-schema_registry_validation"]
+===== `schema_registry_validation`
+
+  * Value can be either of: `auto`, `skip`
+  * Default value is `"auto"`
+
+NOTE: Under most circumstances, the default setting of `auto` should not need to be changed.
+
+When using the schema registry, by default the plugin checks connectivity and validates the schema registry, during plugin registration, before events are processed.
+In some circumstances, this process may fail when it tries to validate an authenticated schema registry, causing the plugin to crash.
+This setting allows the plugin to skip validation during registration, which allows the plugin to continue and events to be processed. Note that an incorrectly configured schema registry will still stop the plugin from processing events.
+
 [id="plugins-{type}s-{plugin}-security_protocol"]
 ===== `security_protocol` 
 

data/docs/output-kafka.asciidoc

@@ -64,6 +64,8 @@ https://kafka.apache.org/{kafka_client_doc}/documentation.html#theproducer
 Kafka producer configuration:
 https://kafka.apache.org/{kafka_client_doc}/documentation.html#producerconfigs
 
+NOTE:  This plugin does not support using a proxy when communicating to the Kafka broker.
+
 [id="plugins-{type}s-{plugin}-options"]
 ==== Kafka Output Configuration Options
 

data/lib/logstash/inputs/kafka.rb

@@ -433,13 +433,16 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
       if schema_registry_url
         props.put(kafka::VALUE_DESERIALIZER_CLASS_CONFIG, Java::io.confluent.kafka.serializers.KafkaAvroDeserializer.java_class)
         serdes_config = Java::io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig
-        props.put(serdes_config::SCHEMA_REGISTRY_URL_CONFIG, schema_registry_url.to_s)
+        props.put(serdes_config::SCHEMA_REGISTRY_URL_CONFIG, schema_registry_url.uri.to_s)
         if schema_registry_proxy && !schema_registry_proxy.empty?
           props.put(serdes_config::PROXY_HOST, @schema_registry_proxy_host)
           props.put(serdes_config::PROXY_PORT, @schema_registry_proxy_port)
         end
         if schema_registry_key && !schema_registry_key.empty?
+          props.put(serdes_config::BASIC_AUTH_CREDENTIALS_SOURCE, 'USER_INFO')
           props.put(serdes_config::USER_INFO_CONFIG, schema_registry_key + ":" + schema_registry_secret.value)
+        else
+          props.put(serdes_config::BASIC_AUTH_CREDENTIALS_SOURCE, 'URL')
         end
       end
       if security_protocol == "SSL"

data/lib/logstash/plugin_mixins/common.rb

@@ -22,6 +22,10 @@ module LogStash
         # Option to set the proxy of the Schema Registry.
         # This option permits to define a proxy to be used to reach the schema registry service instance.
         config :schema_registry_proxy, :validate => :uri
+
+        # Option to skip validating the schema registry during registration. This can be useful when using
+        # certificate based auth
+        config :schema_registry_validation, :validate => ['auto', 'skip'], :default => 'auto'
       end
 
       def check_schema_registry_parameters
@@ -29,10 +33,21 @@ module LogStash
           check_for_schema_registry_conflicts
           @schema_registry_proxy_host, @schema_registry_proxy_port  = split_proxy_into_host_and_port(schema_registry_proxy)
           check_for_key_and_secret
-          check_for_schema_registry_connectivity_and_subjects
+          check_for_schema_registry_connectivity_and_subjects if schema_registry_validation?
         end
       end
 
+      def schema_registry_validation?
+        return false if schema_registry_validation.to_s == 'skip'
+        return false if using_kerberos? # pre-validation doesn't support kerberos
+        
+        true
+      end
+
+      def using_kerberos?
+        security_protocol == "SASL_PLAINTEXT" || security_protocol == "SASL_SSL"
+      end
+
       private
       def check_for_schema_registry_conflicts
         if @value_deserializer_class != LogStash::Inputs::Kafka::DEFAULT_DESERIALIZER_CLASS
@@ -53,9 +68,8 @@ module LogStash
           options[:auth] = {:user => schema_registry_key, :password => schema_registry_secret.value}
         end
         client = Manticore::Client.new(options)
-
         begin
-          response = client.get(@schema_registry_url.to_s + '/subjects').body
+          response = client.get(@schema_registry_url.uri.to_s + '/subjects').body
         rescue Manticore::ManticoreException => e
           raise LogStash::ConfigurationError.new("Schema registry service doesn't respond, error: #{e.message}")
         end

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '10.7.6'
+  s.version         = '10.8.2'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+

data/spec/fixtures/jaas.config

@@ -0,0 +1,5 @@
+SchemaRegistry-Props {
+  org.eclipse.jetty.jaas.spi.PropertyFileLoginModule required
+  file="build/confluent_platform/etc/schema-registry/pwd"
+  debug="true";
+};

data/spec/fixtures/pwd

@@ -0,0 +1,5 @@
+fred: OBF:1w8t1tvf1w261w8v1w1c1tvn1w8x,user,admin
+barney: changeme,user,developer
+admin:admin,admin
+betty: MD5:164c88b302622e17050af52c89945d44,user
+wilma: CRYPT:adpexzg3FUZAk,admin,sr-user

data/spec/integration/inputs/kafka_spec.rb

@@ -206,6 +206,16 @@ end
 
 
 describe "schema registry connection options" do
+  schema_registry = Manticore::Client.new
+  before (:all) do
+    shutdown_schema_registry
+    startup_schema_registry(schema_registry)
+  end
+
+  after(:all) do
+    shutdown_schema_registry
+  end
+
   context "remote endpoint validation" do
     it "should fail if not reachable" do
       config = {'schema_registry_url' => 'http://localnothost:8081'}
@@ -232,8 +242,7 @@ describe "schema registry connection options" do
       end
 
       after(:each) do
-        schema_registry_client = Manticore::Client.new
-        delete_remote_schema(schema_registry_client, SUBJECT_NAME)
+        delete_remote_schema(schema_registry, SUBJECT_NAME)
       end
 
       it "should correctly complete registration phase" do
@@ -264,9 +273,25 @@ end
 
 # AdminClientConfig = org.alpache.kafka.clients.admin.AdminClientConfig
 
+def startup_schema_registry(schema_registry, auth=false)
+  system('./stop_schema_registry.sh')
+  auth ? system('./start_auth_schema_registry.sh') : system('./start_schema_registry.sh')
+  url = auth ? "http://barney:changeme@localhost:8081" : "http://localhost:8081"
+  Stud.try(20.times, [Manticore::SocketException, StandardError, RSpec::Expectations::ExpectationNotMetError]) do
+    expect(schema_registry.get(url).code).to eq(200)
+  end
+end
+
 describe "Schema registry API", :integration => true do
+  schema_registry = Manticore::Client.new
+
+  before(:all) do
+    startup_schema_registry(schema_registry)
+  end
 
-  let(:schema_registry) { Manticore::Client.new }
+  after(:all) do
+    shutdown_schema_registry
+  end
 
   context 'listing subject on clean instance' do
     it "should return an empty set" do
@@ -292,37 +317,58 @@ describe "Schema registry API", :integration => true do
       expect( subjects ).to be_empty
     end
   end
+end
+
+def shutdown_schema_registry
+  system('./stop_schema_registry.sh')
+end
+
+describe "Deserializing with the schema registry", :integration => true do
+  schema_registry = Manticore::Client.new
+
+  shared_examples 'it reads from a topic using a schema registry' do |with_auth|
+
+    before(:all) do
+      shutdown_schema_registry
+      startup_schema_registry(schema_registry, with_auth)
+    end
+
+    after(:all) do
+      shutdown_schema_registry
+    end
 
-  context 'use the schema to serialize' do
     after(:each) do
-      expect( schema_registry.delete('http://localhost:8081/subjects/topic_avro-value').code ).to be(200)
+      expect( schema_registry.delete("#{subject_url}/#{avro_topic_name}-value").code ).to be(200)
       sleep 1
-      expect( schema_registry.delete('http://localhost:8081/subjects/topic_avro-value?permanent=true').code ).to be(200)
+      expect( schema_registry.delete("#{subject_url}/#{avro_topic_name}-value?permanent=true").code ).to be(200)
 
       Stud.try(3.times, [StandardError, RSpec::Expectations::ExpectationNotMetError]) do
         wait(10).for do
-          subjects = JSON.parse schema_registry.get('http://localhost:8081/subjects').body
+          subjects = JSON.parse schema_registry.get(subject_url).body
           subjects.empty?
         end.to be_truthy
       end
     end
 
-    let(:group_id_1) {rand(36**8).to_s(36)}
-
-    let(:avro_topic_name) { "topic_avro" }
-
-    let(:plain_config) do
-      { 'schema_registry_url' => 'http://localhost:8081',
-        'topics' => [avro_topic_name],
-        'codec' => 'plain',
-        'group_id' => group_id_1,
-        'auto_offset_reset' => 'earliest' }
+    let(:base_config) do
+      {
+          'topics' => [avro_topic_name],
+          'codec' => 'plain',
+          'group_id' => group_id_1,
+          'auto_offset_reset' => 'earliest'
+      }
     end
 
-    def delete_topic_if_exists(topic_name)
+    let(:group_id_1) {rand(36**8).to_s(36)}
+
+    def delete_topic_if_exists(topic_name, user = nil, password = nil)
       props = java.util.Properties.new
       props.put(Java::org.apache.kafka.clients.admin.AdminClientConfig::BOOTSTRAP_SERVERS_CONFIG, "localhost:9092")
-
+      serdes_config = Java::io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig
+      unless user.nil?
+        props.put(serdes_config::BASIC_AUTH_CREDENTIALS_SOURCE, 'USER_INFO')
+        props.put(serdes_config::USER_INFO_CONFIG,  "#{user}:#{password}")
+      end
       admin_client = org.apache.kafka.clients.admin.AdminClient.create(props)
       topics_list = admin_client.listTopics().names().get()
       if topics_list.contains(topic_name)
@@ -331,7 +377,7 @@ describe "Schema registry API", :integration => true do
       end
     end
 
-    def write_some_data_to(topic_name)
+    def write_some_data_to(topic_name, user = nil, password = nil)
       props = java.util.Properties.new
       config = org.apache.kafka.clients.producer.ProducerConfig
 
@@ -339,6 +385,10 @@ describe "Schema registry API", :integration => true do
       props.put(serdes_config::SCHEMA_REGISTRY_URL_CONFIG, "http://localhost:8081")
 
       props.put(config::BOOTSTRAP_SERVERS_CONFIG, "localhost:9092")
+      unless user.nil?
+        props.put(serdes_config::BASIC_AUTH_CREDENTIALS_SOURCE, 'USER_INFO')
+        props.put(serdes_config::USER_INFO_CONFIG,  "#{user}:#{password}")
+      end
       props.put(config::KEY_SERIALIZER_CLASS_CONFIG, org.apache.kafka.common.serialization.StringSerializer.java_class)
       props.put(config::VALUE_SERIALIZER_CLASS_CONFIG, Java::io.confluent.kafka.serializers.KafkaAvroSerializer.java_class)
 
@@ -360,11 +410,11 @@ describe "Schema registry API", :integration => true do
     end
 
     it "stored a new schema using Avro Kafka serdes" do
-      delete_topic_if_exists avro_topic_name
-      write_some_data_to avro_topic_name
+      auth ? delete_topic_if_exists(avro_topic_name, user, password) : delete_topic_if_exists(avro_topic_name)
+      auth ? write_some_data_to(avro_topic_name, user, password) : write_some_data_to(avro_topic_name)
 
-      subjects = JSON.parse schema_registry.get('http://localhost:8081/subjects').body
-      expect( subjects ).to contain_exactly("topic_avro-value")
+      subjects = JSON.parse schema_registry.get(subject_url).body
+      expect( subjects ).to contain_exactly("#{avro_topic_name}-value")
 
       num_events = 1
       queue = consume_messages(plain_config, timeout: 30, event_count: num_events)
@@ -375,4 +425,43 @@ describe "Schema registry API", :integration => true do
       expect( elem.get("map_field")["inner_field"] ).to eq("inner value")
     end
   end
+
+  context 'with an unauthed schema registry' do
+    let(:auth) { false }
+    let(:avro_topic_name) { "topic_avro" }
+    let(:subject_url) { "http://localhost:8081/subjects" }
+    let(:plain_config)  { base_config.merge!({'schema_registry_url' => "http://localhost:8081"}) }
+
+    it_behaves_like 'it reads from a topic using a schema registry', false
+  end
+
+  context 'with an authed schema registry' do
+    let(:auth) { true }
+    let(:user) { "barney" }
+    let(:password) { "changeme" }
+    let(:avro_topic_name) { "topic_avro_auth" }
+    let(:subject_url) { "http://#{user}:#{password}@localhost:8081/subjects" }
+
+    context 'using schema_registry_key' do
+      let(:plain_config) do
+        base_config.merge!({
+          'schema_registry_url' => "http://localhost:8081",
+          'schema_registry_key' => user,
+          'schema_registry_secret' => password
+        })
+      end
+
+      it_behaves_like 'it reads from a topic using a schema registry', true
+    end
+
+    context 'using schema_registry_url' do
+      let(:plain_config) do
+        base_config.merge!({
+          'schema_registry_url' => "http://#{user}:#{password}@localhost:8081"
+        })
+      end
+
+      it_behaves_like 'it reads from a topic using a schema registry', true
+    end
+  end
 end

data/spec/unit/inputs/kafka_spec.rb

@@ -177,11 +177,16 @@ describe LogStash::Inputs::Kafka do
     end
   end
 
-  context "register parameter verification" do
+  describe "schema registry parameter verification" do
+    let(:base_config) do {
+          'schema_registry_url' => 'http://localhost:8081',
+          'topics' => ['logstash'],
+          'consumer_threads' => 4
+    }
+    end
+
     context "schema_registry_url" do
-      let(:config) do
-        { 'schema_registry_url' => 'http://localhost:8081', 'topics' => ['logstash'], 'consumer_threads' => 4 }
-      end
+     let(:config) { base_config }
 
       it "conflict with value_deserializer_class should fail" do
         config['value_deserializer_class'] = 'my.fantasy.Deserializer'
@@ -194,19 +199,63 @@ describe LogStash::Inputs::Kafka do
       end
     end
 
-    context "decorate_events" do
-      let(:config) { { 'decorate_events' => 'extended'} }
+    context 'when kerberos auth is used' do
+      ['SASL_SSL', 'SASL_PLAINTEXT'].each do |protocol|
+        context "with #{protocol}" do
+          ['auto', 'skip'].each do |vsr|
+            context "when validata_schema_registry is #{vsr}" do
+              let(:config) { base_config.merge({'security_protocol' => protocol,
+                                                'schema_registry_validation' => vsr})
+              }
+              it 'skips verification' do
+                expect(subject).not_to receive(:check_for_schema_registry_connectivity_and_subjects)
+                expect { subject.register }.not_to raise_error
+              end
+            end
+          end
+        end
+      end
+    end
 
-      it "should raise error for invalid value" do
-        config['decorate_events'] = 'avoid'
-        expect { subject.register }.to raise_error LogStash::ConfigurationError, /Something is wrong with your configuration./
+    context 'when kerberos auth is not used' do
+      context "when skip_verify is set to auto" do
+        let(:config) { base_config.merge({'schema_registry_validation' => 'auto'})}
+        it 'performs verification' do
+          expect(subject).to receive(:check_for_schema_registry_connectivity_and_subjects)
+          expect { subject.register }.not_to raise_error
+        end
       end
 
-      it "should map old true boolean value to :record_props mode" do
-        config['decorate_events'] = "true"
-        subject.register
-        expect(subject.metadata_mode).to include(:record_props)
+      context "when skip_verify is set to default" do
+        let(:config) { base_config }
+        it 'performs verification' do
+          expect(subject).to receive(:check_for_schema_registry_connectivity_and_subjects)
+          expect { subject.register }.not_to raise_error
+        end
       end
+
+      context "when skip_verify is set to skip" do
+        let(:config) { base_config.merge({'schema_registry_validation' => 'skip'})}
+        it 'should skip verification' do
+          expect(subject).not_to receive(:check_for_schema_registry_connectivity_and_subjects)
+          expect { subject.register }.not_to raise_error
+        end
+      end
+    end
+  end
+
+  context "decorate_events" do
+    let(:config) { { 'decorate_events' => 'extended'} }
+
+    it "should raise error for invalid value" do
+      config['decorate_events'] = 'avoid'
+      expect { subject.register }.to raise_error LogStash::ConfigurationError, /Something is wrong with your configuration./
+    end
+
+    it "should map old true boolean value to :record_props mode" do
+      config['decorate_events'] = "true"
+      subject.register
+      expect(subject.metadata_mode).to include(:record_props)
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-kafka
 version: !ruby/object:Gem::Version
-  version: 10.7.6
+  version: 10.8.2
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-05-31 00:00:00.000000000 Z
+date: 2021-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -235,6 +235,8 @@ files:
 - lib/logstash/plugin_mixins/kafka_support.rb
 - logstash-integration-kafka.gemspec
 - spec/check_docs_spec.rb
+- spec/fixtures/jaas.config
+- spec/fixtures/pwd
 - spec/fixtures/trust-store_stub.jks
 - spec/integration/inputs/kafka_spec.rb
 - spec/integration/outputs/kafka_spec.rb
@@ -278,13 +280,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Integration with Kafka - input and output plugins
 test_files:
 - spec/check_docs_spec.rb
+- spec/fixtures/jaas.config
+- spec/fixtures/pwd
 - spec/fixtures/trust-store_stub.jks
 - spec/integration/inputs/kafka_spec.rb
 - spec/integration/outputs/kafka_spec.rb