logstash-integration-kafka 10.7.7-java → 10.8.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7a9e872c7a49cd1eb9fc9709c10be72d1c9609b0f19ef5550a2dd7cb317a925e
-  data.tar.gz: b0541ab279e7b7fcea74f1a951e7916abf54adde8bae60e2c4b5897b6f4daadb
+  metadata.gz: a322e757007ef9c740f48a93ebe97aa50511a29b17450219ea4f2d08a49934e1
+  data.tar.gz: 111a1d72591d14876f33dba65cfc29e6c9ae397397f295e3a94e5078b413eb3a
 SHA512:
-  metadata.gz: 1e53d939b471032107afaeb2b7656bf6c755d803e1effbda487b953ae6cc0b79fd97458834d079168023e0ecfa8aeec67a474dc1a4a99336a7be10af3d1b6412
-  data.tar.gz: bea64e388923bab253278e46eccc0701c2bda8f3067cc1e429223fbc3f9cd3a797ced2e334755bbbe3bbf1cf711237359d85e1c6ec22cde822ced2285840fd9f
+  metadata.gz: bac61cfb956bb284df97746df68afc90b12b1d04186eba0cf27fc68886858dad965aefcf46566dc8e27e8a484b524c7ce8d3185d24d9412798e06defc9f78ab8
+  data.tar.gz: f706aaad143f31fdd31ea47c5d1604f490f505f08329ecd07dff75201c51d1983701e79e2f74a52f484c860eb026a26880447e3b89a985dffeb7367728cab1ef

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 10.8.0
+  - Added config setting to enable schema registry validation to be skipped when an authentication scheme unsupported
+    by the validator is used [#97](https://github.com/logstash-plugins/logstash-integration-kafka/pull/97)
+
 ## 10.7.7
   - Fix: Correct the settings to allow basic auth to work properly, either by setting `schema_registry_key/secret` or embedding username/password in the
     url [#94](https://github.com/logstash-plugins/logstash-integration-kafka/pull/94)

data/docs/input-kafka.asciidoc

@@ -128,6 +128,7 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai
 | <<plugins-{type}s-{plugin}-schema_registry_proxy>> |<<uri,uri>>|No
 | <<plugins-{type}s-{plugin}-schema_registry_secret>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-schema_registry_url>> |<<uri,uri>>|No
+| <<plugins-{type}s-{plugin}-schema_registry_validation>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-security_protocol>> |<<string,string>>, one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No
 | <<plugins-{type}s-{plugin}-send_buffer_bytes>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-session_timeout_ms>> |<<number,number>>|No
@@ -576,6 +577,18 @@ The schemas must follow a naming convention with the pattern <topic name>-value.
 Use either the Schema Registry config option or the
 <<plugins-{type}s-{plugin}-value_deserializer_class>> config option, but not both.
 
+[id="plugins-{type}s-{plugin}-schema_registry_validation"]
+===== `schema_registry_validation`
+
+  * Value can be either of: `auto`, `skip`
+  * Default value is `"auto"`
+
+NOTE: Under most circumstances, the default setting of `auto` should not need to be changed.
+
+When using the schema registry, by default the plugin checks connectivity and validates the schema registry, during plugin registration, before events are processed.
+In some circumstances, this process may fail when it tries to validate an authenticated schema registry, causing the plugin to crash.
+This setting allows the plugin to skip validation during registration, which allows the plugin to continue and events to be processed. Note that an incorrectly configured schema registry will still stop the plugin from processing events.
+
 [id="plugins-{type}s-{plugin}-security_protocol"]
 ===== `security_protocol` 
 

data/lib/logstash/plugin_mixins/common.rb

@@ -22,6 +22,10 @@ module LogStash
         # Option to set the proxy of the Schema Registry.
         # This option permits to define a proxy to be used to reach the schema registry service instance.
         config :schema_registry_proxy, :validate => :uri
+
+        # Option to skip validating the schema registry during registration. This can be useful when using
+        # certificate based auth
+        config :schema_registry_validation, :validate => ['auto', 'skip'], :default => 'auto'
       end
 
       def check_schema_registry_parameters
@@ -29,10 +33,21 @@ module LogStash
           check_for_schema_registry_conflicts
           @schema_registry_proxy_host, @schema_registry_proxy_port  = split_proxy_into_host_and_port(schema_registry_proxy)
           check_for_key_and_secret
-          check_for_schema_registry_connectivity_and_subjects
+          check_for_schema_registry_connectivity_and_subjects if schema_registry_validation?
         end
       end
 
+      def schema_registry_validation?
+        return false if schema_registry_validation.to_s == 'skip'
+        return false if using_kerberos? # pre-validation doesn't support kerberos
+        
+        true
+      end
+
+      def using_kerberos?
+        security_protocol == "SASL_PLAINTEXT" || security_protocol == "SASL_SSL"
+      end
+
       private
       def check_for_schema_registry_conflicts
         if @value_deserializer_class != LogStash::Inputs::Kafka::DEFAULT_DESERIALIZER_CLASS

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '10.7.7'
+  s.version         = '10.8.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+

data/spec/unit/inputs/kafka_spec.rb

@@ -177,11 +177,16 @@ describe LogStash::Inputs::Kafka do
     end
   end
 
-  context "register parameter verification" do
+  describe "schema registry parameter verification" do
+    let(:base_config) do {
+          'schema_registry_url' => 'http://localhost:8081',
+          'topics' => ['logstash'],
+          'consumer_threads' => 4
+    }
+    end
+
     context "schema_registry_url" do
-      let(:config) do
-        { 'schema_registry_url' => 'http://localhost:8081', 'topics' => ['logstash'], 'consumer_threads' => 4 }
-      end
+     let(:config) { base_config }
 
       it "conflict with value_deserializer_class should fail" do
         config['value_deserializer_class'] = 'my.fantasy.Deserializer'
@@ -194,19 +199,63 @@ describe LogStash::Inputs::Kafka do
       end
     end
 
-    context "decorate_events" do
-      let(:config) { { 'decorate_events' => 'extended'} }
+    context 'when kerberos auth is used' do
+      ['SASL_SSL', 'SASL_PLAINTEXT'].each do |protocol|
+        context "with #{protocol}" do
+          ['auto', 'skip'].each do |vsr|
+            context "when validata_schema_registry is #{vsr}" do
+              let(:config) { base_config.merge({'security_protocol' => protocol,
+                                                'schema_registry_validation' => vsr})
+              }
+              it 'skips verification' do
+                expect(subject).not_to receive(:check_for_schema_registry_connectivity_and_subjects)
+                expect { subject.register }.not_to raise_error
+              end
+            end
+          end
+        end
+      end
+    end
 
-      it "should raise error for invalid value" do
-        config['decorate_events'] = 'avoid'
-        expect { subject.register }.to raise_error LogStash::ConfigurationError, /Something is wrong with your configuration./
+    context 'when kerberos auth is not used' do
+      context "when skip_verify is set to auto" do
+        let(:config) { base_config.merge({'schema_registry_validation' => 'auto'})}
+        it 'performs verification' do
+          expect(subject).to receive(:check_for_schema_registry_connectivity_and_subjects)
+          expect { subject.register }.not_to raise_error
+        end
       end
 
-      it "should map old true boolean value to :record_props mode" do
-        config['decorate_events'] = "true"
-        subject.register
-        expect(subject.metadata_mode).to include(:record_props)
+      context "when skip_verify is set to default" do
+        let(:config) { base_config }
+        it 'performs verification' do
+          expect(subject).to receive(:check_for_schema_registry_connectivity_and_subjects)
+          expect { subject.register }.not_to raise_error
+        end
       end
+
+      context "when skip_verify is set to skip" do
+        let(:config) { base_config.merge({'schema_registry_validation' => 'skip'})}
+        it 'should skip verification' do
+          expect(subject).not_to receive(:check_for_schema_registry_connectivity_and_subjects)
+          expect { subject.register }.not_to raise_error
+        end
+      end
+    end
+  end
+
+  context "decorate_events" do
+    let(:config) { { 'decorate_events' => 'extended'} }
+
+    it "should raise error for invalid value" do
+      config['decorate_events'] = 'avoid'
+      expect { subject.register }.to raise_error LogStash::ConfigurationError, /Something is wrong with your configuration./
+    end
+
+    it "should map old true boolean value to :record_props mode" do
+      config['decorate_events'] = "true"
+      subject.register
+      expect(subject.metadata_mode).to include(:record_props)
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-kafka
 version: !ruby/object:Gem::Version
-  version: 10.7.7
+  version: 10.8.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-07-06 00:00:00.000000000 Z
+date: 2021-07-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement