logstash-integration-kafka 10.7.5-java → 10.8.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5e76697fe666ea555f7256a8a42865a749ee00e05f9e7294d89c481b1ba6a7c8
-  data.tar.gz: 5cfffe44f6e36776efb87a77e3a30ac03cdc802ae769aa423abdae4e0a36a6d5
+  metadata.gz: ff8065a6958598d8e33ecd2f78f27d7ec2c0ce1334516966d41a4426d437a4bb
+  data.tar.gz: f5a04ae25fc6394751fd63ff0c76171db82b7fccdd9962d35c94c06f8b516c39
 SHA512:
-  metadata.gz: 9052d2ae8570e274840882751383a0cfb3c76e6858ce0519ccfdf2be8ca773b0becc9c27e1ec0f52319004b5ee21e4fdd32d60affed719f5b54b4772b4329541
-  data.tar.gz: ff4924b87505befa7f19ddfc2699c109fbdcd53c313b4b7cb5775b7fe6de2853cc98480fae1c9267463e88eac3dbf8c2484178af803505fb820aa0b26af4e5e9
+  metadata.gz: d6a2630bb76e853e185c02933fe065be84e6d0f863e255e611e67b9de772b5661d09cf74fb8ba0e1df035e2742a9fbf73a3b00f2bb87801414e65d5043c9491c
+  data.tar.gz: e6ff413abd74c78f463d95f7c476fdecc2005cee804f426e56056e66300a785cf0789f73b5aa1d5aed4966d42a34a007b52361744612ecb5420891ff493c0df0

data/CHANGELOG.md

@@ -1,3 +1,17 @@
+## 10.8.1
+  - [DOC] Removed a setting recommendation that is no longer applicable for Kafka 2.0+ [#99](https://github.com/logstash-plugins/logstash-integration-kafka/pull/99)
+
+## 10.8.0
+  - Added config setting to enable schema registry validation to be skipped when an authentication scheme unsupported
+    by the validator is used [#97](https://github.com/logstash-plugins/logstash-integration-kafka/pull/97)
+
+## 10.7.7
+  - Fix: Correct the settings to allow basic auth to work properly, either by setting `schema_registry_key/secret` or embedding username/password in the
+    url [#94](https://github.com/logstash-plugins/logstash-integration-kafka/pull/94)
+
+## 10.7.6
+  - Test: specify development dependency version [#91](https://github.com/logstash-plugins/logstash-integration-kafka/pull/91)
+
 ## 10.7.5
   - Improved error handling in the input plugin to avoid errors 'escaping' from the plugin, and crashing the logstash
     process [#87](https://github.com/logstash-plugins/logstash-integration-kafka/pull/87)

data/docs/input-kafka.asciidoc

@@ -128,6 +128,7 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai
 | <<plugins-{type}s-{plugin}-schema_registry_proxy>> |<<uri,uri>>|No
 | <<plugins-{type}s-{plugin}-schema_registry_secret>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-schema_registry_url>> |<<uri,uri>>|No
+| <<plugins-{type}s-{plugin}-schema_registry_validation>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-security_protocol>> |<<string,string>>, one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No
 | <<plugins-{type}s-{plugin}-send_buffer_bytes>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-session_timeout_ms>> |<<number,number>>|No
@@ -414,7 +415,6 @@ The maximum delay between invocations of poll() when using consumer group manage
 an upper bound on the amount of time that the consumer can be idle before fetching more records. 
 If poll() is not called before expiration of this timeout, then the consumer is considered failed and 
 the group will rebalance in order to reassign the partitions to another member.
-The value of the configuration `request_timeout_ms` must always be larger than `max_poll_interval_ms`. ???
 
 [id="plugins-{type}s-{plugin}-max_poll_records"]
 ===== `max_poll_records` 
@@ -576,6 +576,18 @@ The schemas must follow a naming convention with the pattern <topic name>-value.
 Use either the Schema Registry config option or the
 <<plugins-{type}s-{plugin}-value_deserializer_class>> config option, but not both.
 
+[id="plugins-{type}s-{plugin}-schema_registry_validation"]
+===== `schema_registry_validation`
+
+  * Value can be either of: `auto`, `skip`
+  * Default value is `"auto"`
+
+NOTE: Under most circumstances, the default setting of `auto` should not need to be changed.
+
+When using the schema registry, by default the plugin checks connectivity and validates the schema registry, during plugin registration, before events are processed.
+In some circumstances, this process may fail when it tries to validate an authenticated schema registry, causing the plugin to crash.
+This setting allows the plugin to skip validation during registration, which allows the plugin to continue and events to be processed. Note that an incorrectly configured schema registry will still stop the plugin from processing events.
+
 [id="plugins-{type}s-{plugin}-security_protocol"]
 ===== `security_protocol` 
 

data/lib/logstash/inputs/kafka.rb

@@ -433,13 +433,16 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
       if schema_registry_url
         props.put(kafka::VALUE_DESERIALIZER_CLASS_CONFIG, Java::io.confluent.kafka.serializers.KafkaAvroDeserializer.java_class)
         serdes_config = Java::io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig
-        props.put(serdes_config::SCHEMA_REGISTRY_URL_CONFIG, schema_registry_url.to_s)
+        props.put(serdes_config::SCHEMA_REGISTRY_URL_CONFIG, schema_registry_url.uri.to_s)
         if schema_registry_proxy && !schema_registry_proxy.empty?
           props.put(serdes_config::PROXY_HOST, @schema_registry_proxy_host)
           props.put(serdes_config::PROXY_PORT, @schema_registry_proxy_port)
         end
         if schema_registry_key && !schema_registry_key.empty?
+          props.put(serdes_config::BASIC_AUTH_CREDENTIALS_SOURCE, 'USER_INFO')
           props.put(serdes_config::USER_INFO_CONFIG, schema_registry_key + ":" + schema_registry_secret.value)
+        else
+          props.put(serdes_config::BASIC_AUTH_CREDENTIALS_SOURCE, 'URL')
         end
       end
       if security_protocol == "SSL"

data/lib/logstash/plugin_mixins/common.rb

@@ -22,6 +22,10 @@ module LogStash
         # Option to set the proxy of the Schema Registry.
         # This option permits to define a proxy to be used to reach the schema registry service instance.
         config :schema_registry_proxy, :validate => :uri
+
+        # Option to skip validating the schema registry during registration. This can be useful when using
+        # certificate based auth
+        config :schema_registry_validation, :validate => ['auto', 'skip'], :default => 'auto'
       end
 
       def check_schema_registry_parameters
@@ -29,10 +33,21 @@ module LogStash
           check_for_schema_registry_conflicts
           @schema_registry_proxy_host, @schema_registry_proxy_port  = split_proxy_into_host_and_port(schema_registry_proxy)
           check_for_key_and_secret
-          check_for_schema_registry_connectivity_and_subjects
+          check_for_schema_registry_connectivity_and_subjects if schema_registry_validation?
         end
       end
 
+      def schema_registry_validation?
+        return false if schema_registry_validation.to_s == 'skip'
+        return false if using_kerberos? # pre-validation doesn't support kerberos
+        
+        true
+      end
+
+      def using_kerberos?
+        security_protocol == "SASL_PLAINTEXT" || security_protocol == "SASL_SSL"
+      end
+
       private
       def check_for_schema_registry_conflicts
         if @value_deserializer_class != LogStash::Inputs::Kafka::DEFAULT_DESERIALIZER_CLASS
@@ -53,9 +68,8 @@ module LogStash
           options[:auth] = {:user => schema_registry_key, :password => schema_registry_secret.value}
         end
         client = Manticore::Client.new(options)
-
         begin
-          response = client.get(@schema_registry_url.to_s + '/subjects').body
+          response = client.get(@schema_registry_url.uri.to_s + '/subjects').body
         rescue Manticore::ManticoreException => e
           raise LogStash::ConfigurationError.new("Schema registry service doesn't respond, error: #{e.message}")
         end

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '10.7.5'
+  s.version         = '10.8.1'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+
@@ -51,6 +51,7 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'rspec-wait'
-  s.add_development_dependency 'ruby-kafka'
+  s.add_development_dependency 'digest-crc', '~> 0.5.1' # 0.6.0 started using a C-ext
+  s.add_development_dependency 'ruby-kafka' # depends on digest-crc
   s.add_development_dependency 'snappy'
 end

data/spec/fixtures/jaas.config

@@ -0,0 +1,5 @@
+SchemaRegistry-Props {
+  org.eclipse.jetty.jaas.spi.PropertyFileLoginModule required
+  file="build/confluent_platform/etc/schema-registry/pwd"
+  debug="true";
+};

data/spec/fixtures/pwd

@@ -0,0 +1,5 @@
+fred: OBF:1w8t1tvf1w261w8v1w1c1tvn1w8x,user,admin
+barney: changeme,user,developer
+admin:admin,admin
+betty: MD5:164c88b302622e17050af52c89945d44,user
+wilma: CRYPT:adpexzg3FUZAk,admin,sr-user

data/spec/integration/inputs/kafka_spec.rb

@@ -206,6 +206,16 @@ end
 
 
 describe "schema registry connection options" do
+  schema_registry = Manticore::Client.new
+  before (:all) do
+    shutdown_schema_registry
+    startup_schema_registry(schema_registry)
+  end
+
+  after(:all) do
+    shutdown_schema_registry
+  end
+
   context "remote endpoint validation" do
     it "should fail if not reachable" do
       config = {'schema_registry_url' => 'http://localnothost:8081'}
@@ -232,8 +242,7 @@ describe "schema registry connection options" do
       end
 
       after(:each) do
-        schema_registry_client = Manticore::Client.new
-        delete_remote_schema(schema_registry_client, SUBJECT_NAME)
+        delete_remote_schema(schema_registry, SUBJECT_NAME)
       end
 
       it "should correctly complete registration phase" do
@@ -264,9 +273,25 @@ end
 
 # AdminClientConfig = org.alpache.kafka.clients.admin.AdminClientConfig
 
+def startup_schema_registry(schema_registry, auth=false)
+  system('./stop_schema_registry.sh')
+  auth ? system('./start_auth_schema_registry.sh') : system('./start_schema_registry.sh')
+  url = auth ? "http://barney:changeme@localhost:8081" : "http://localhost:8081"
+  Stud.try(20.times, [Manticore::SocketException, StandardError, RSpec::Expectations::ExpectationNotMetError]) do
+    expect(schema_registry.get(url).code).to eq(200)
+  end
+end
+
 describe "Schema registry API", :integration => true do
+  schema_registry = Manticore::Client.new
+
+  before(:all) do
+    startup_schema_registry(schema_registry)
+  end
 
-  let(:schema_registry) { Manticore::Client.new }
+  after(:all) do
+    shutdown_schema_registry
+  end
 
   context 'listing subject on clean instance' do
     it "should return an empty set" do
@@ -292,37 +317,58 @@ describe "Schema registry API", :integration => true do
       expect( subjects ).to be_empty
     end
   end
+end
+
+def shutdown_schema_registry
+  system('./stop_schema_registry.sh')
+end
+
+describe "Deserializing with the schema registry", :integration => true do
+  schema_registry = Manticore::Client.new
+
+  shared_examples 'it reads from a topic using a schema registry' do |with_auth|
+
+    before(:all) do
+      shutdown_schema_registry
+      startup_schema_registry(schema_registry, with_auth)
+    end
+
+    after(:all) do
+      shutdown_schema_registry
+    end
 
-  context 'use the schema to serialize' do
     after(:each) do
-      expect( schema_registry.delete('http://localhost:8081/subjects/topic_avro-value').code ).to be(200)
+      expect( schema_registry.delete("#{subject_url}/#{avro_topic_name}-value").code ).to be(200)
       sleep 1
-      expect( schema_registry.delete('http://localhost:8081/subjects/topic_avro-value?permanent=true').code ).to be(200)
+      expect( schema_registry.delete("#{subject_url}/#{avro_topic_name}-value?permanent=true").code ).to be(200)
 
       Stud.try(3.times, [StandardError, RSpec::Expectations::ExpectationNotMetError]) do
         wait(10).for do
-          subjects = JSON.parse schema_registry.get('http://localhost:8081/subjects').body
+          subjects = JSON.parse schema_registry.get(subject_url).body
           subjects.empty?
         end.to be_truthy
       end
     end
 
-    let(:group_id_1) {rand(36**8).to_s(36)}
-
-    let(:avro_topic_name) { "topic_avro" }
-
-    let(:plain_config) do
-      { 'schema_registry_url' => 'http://localhost:8081',
-        'topics' => [avro_topic_name],
-        'codec' => 'plain',
-        'group_id' => group_id_1,
-        'auto_offset_reset' => 'earliest' }
+    let(:base_config) do
+      {
+          'topics' => [avro_topic_name],
+          'codec' => 'plain',
+          'group_id' => group_id_1,
+          'auto_offset_reset' => 'earliest'
+      }
     end
 
-    def delete_topic_if_exists(topic_name)
+    let(:group_id_1) {rand(36**8).to_s(36)}
+
+    def delete_topic_if_exists(topic_name, user = nil, password = nil)
       props = java.util.Properties.new
       props.put(Java::org.apache.kafka.clients.admin.AdminClientConfig::BOOTSTRAP_SERVERS_CONFIG, "localhost:9092")
-
+      serdes_config = Java::io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig
+      unless user.nil?
+        props.put(serdes_config::BASIC_AUTH_CREDENTIALS_SOURCE, 'USER_INFO')
+        props.put(serdes_config::USER_INFO_CONFIG,  "#{user}:#{password}")
+      end
       admin_client = org.apache.kafka.clients.admin.AdminClient.create(props)
       topics_list = admin_client.listTopics().names().get()
       if topics_list.contains(topic_name)
@@ -331,7 +377,7 @@ describe "Schema registry API", :integration => true do
       end
     end
 
-    def write_some_data_to(topic_name)
+    def write_some_data_to(topic_name, user = nil, password = nil)
       props = java.util.Properties.new
       config = org.apache.kafka.clients.producer.ProducerConfig
 
@@ -339,6 +385,10 @@ describe "Schema registry API", :integration => true do
       props.put(serdes_config::SCHEMA_REGISTRY_URL_CONFIG, "http://localhost:8081")
 
       props.put(config::BOOTSTRAP_SERVERS_CONFIG, "localhost:9092")
+      unless user.nil?
+        props.put(serdes_config::BASIC_AUTH_CREDENTIALS_SOURCE, 'USER_INFO')
+        props.put(serdes_config::USER_INFO_CONFIG,  "#{user}:#{password}")
+      end
       props.put(config::KEY_SERIALIZER_CLASS_CONFIG, org.apache.kafka.common.serialization.StringSerializer.java_class)
       props.put(config::VALUE_SERIALIZER_CLASS_CONFIG, Java::io.confluent.kafka.serializers.KafkaAvroSerializer.java_class)
 
@@ -360,11 +410,11 @@ describe "Schema registry API", :integration => true do
     end
 
     it "stored a new schema using Avro Kafka serdes" do
-      delete_topic_if_exists avro_topic_name
-      write_some_data_to avro_topic_name
+      auth ? delete_topic_if_exists(avro_topic_name, user, password) : delete_topic_if_exists(avro_topic_name)
+      auth ? write_some_data_to(avro_topic_name, user, password) : write_some_data_to(avro_topic_name)
 
-      subjects = JSON.parse schema_registry.get('http://localhost:8081/subjects').body
-      expect( subjects ).to contain_exactly("topic_avro-value")
+      subjects = JSON.parse schema_registry.get(subject_url).body
+      expect( subjects ).to contain_exactly("#{avro_topic_name}-value")
 
       num_events = 1
       queue = consume_messages(plain_config, timeout: 30, event_count: num_events)
@@ -375,4 +425,43 @@ describe "Schema registry API", :integration => true do
       expect( elem.get("map_field")["inner_field"] ).to eq("inner value")
     end
   end
+
+  context 'with an unauthed schema registry' do
+    let(:auth) { false }
+    let(:avro_topic_name) { "topic_avro" }
+    let(:subject_url) { "http://localhost:8081/subjects" }
+    let(:plain_config)  { base_config.merge!({'schema_registry_url' => "http://localhost:8081"}) }
+
+    it_behaves_like 'it reads from a topic using a schema registry', false
+  end
+
+  context 'with an authed schema registry' do
+    let(:auth) { true }
+    let(:user) { "barney" }
+    let(:password) { "changeme" }
+    let(:avro_topic_name) { "topic_avro_auth" }
+    let(:subject_url) { "http://#{user}:#{password}@localhost:8081/subjects" }
+
+    context 'using schema_registry_key' do
+      let(:plain_config) do
+        base_config.merge!({
+          'schema_registry_url' => "http://localhost:8081",
+          'schema_registry_key' => user,
+          'schema_registry_secret' => password
+        })
+      end
+
+      it_behaves_like 'it reads from a topic using a schema registry', true
+    end
+
+    context 'using schema_registry_url' do
+      let(:plain_config) do
+        base_config.merge!({
+          'schema_registry_url' => "http://#{user}:#{password}@localhost:8081"
+        })
+      end
+
+      it_behaves_like 'it reads from a topic using a schema registry', true
+    end
+  end
 end

data/spec/unit/inputs/kafka_spec.rb

@@ -177,11 +177,16 @@ describe LogStash::Inputs::Kafka do
     end
   end
 
-  context "register parameter verification" do
+  describe "schema registry parameter verification" do
+    let(:base_config) do {
+          'schema_registry_url' => 'http://localhost:8081',
+          'topics' => ['logstash'],
+          'consumer_threads' => 4
+    }
+    end
+
     context "schema_registry_url" do
-      let(:config) do
-        { 'schema_registry_url' => 'http://localhost:8081', 'topics' => ['logstash'], 'consumer_threads' => 4 }
-      end
+     let(:config) { base_config }
 
       it "conflict with value_deserializer_class should fail" do
         config['value_deserializer_class'] = 'my.fantasy.Deserializer'
@@ -194,19 +199,63 @@ describe LogStash::Inputs::Kafka do
       end
     end
 
-    context "decorate_events" do
-      let(:config) { { 'decorate_events' => 'extended'} }
+    context 'when kerberos auth is used' do
+      ['SASL_SSL', 'SASL_PLAINTEXT'].each do |protocol|
+        context "with #{protocol}" do
+          ['auto', 'skip'].each do |vsr|
+            context "when validata_schema_registry is #{vsr}" do
+              let(:config) { base_config.merge({'security_protocol' => protocol,
+                                                'schema_registry_validation' => vsr})
+              }
+              it 'skips verification' do
+                expect(subject).not_to receive(:check_for_schema_registry_connectivity_and_subjects)
+                expect { subject.register }.not_to raise_error
+              end
+            end
+          end
+        end
+      end
+    end
 
-      it "should raise error for invalid value" do
-        config['decorate_events'] = 'avoid'
-        expect { subject.register }.to raise_error LogStash::ConfigurationError, /Something is wrong with your configuration./
+    context 'when kerberos auth is not used' do
+      context "when skip_verify is set to auto" do
+        let(:config) { base_config.merge({'schema_registry_validation' => 'auto'})}
+        it 'performs verification' do
+          expect(subject).to receive(:check_for_schema_registry_connectivity_and_subjects)
+          expect { subject.register }.not_to raise_error
+        end
       end
 
-      it "should map old true boolean value to :record_props mode" do
-        config['decorate_events'] = "true"
-        subject.register
-        expect(subject.metadata_mode).to include(:record_props)
+      context "when skip_verify is set to default" do
+        let(:config) { base_config }
+        it 'performs verification' do
+          expect(subject).to receive(:check_for_schema_registry_connectivity_and_subjects)
+          expect { subject.register }.not_to raise_error
+        end
       end
+
+      context "when skip_verify is set to skip" do
+        let(:config) { base_config.merge({'schema_registry_validation' => 'skip'})}
+        it 'should skip verification' do
+          expect(subject).not_to receive(:check_for_schema_registry_connectivity_and_subjects)
+          expect { subject.register }.not_to raise_error
+        end
+      end
+    end
+  end
+
+  context "decorate_events" do
+    let(:config) { { 'decorate_events' => 'extended'} }
+
+    it "should raise error for invalid value" do
+      config['decorate_events'] = 'avoid'
+      expect { subject.register }.to raise_error LogStash::ConfigurationError, /Something is wrong with your configuration./
+    end
+
+    it "should map old true boolean value to :record_props mode" do
+      config['decorate_events'] = "true"
+      subject.register
+      expect(subject.metadata_mode).to include(:record_props)
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-kafka
 version: !ruby/object:Gem::Version
-  version: 10.7.5
+  version: 10.8.1
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-05-26 00:00:00.000000000 Z
+date: 2021-07-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -168,6 +168,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.1
+  name: digest-crc
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.1
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -221,6 +235,8 @@ files:
 - lib/logstash/plugin_mixins/kafka_support.rb
 - logstash-integration-kafka.gemspec
 - spec/check_docs_spec.rb
+- spec/fixtures/jaas.config
+- spec/fixtures/pwd
 - spec/fixtures/trust-store_stub.jks
 - spec/integration/inputs/kafka_spec.rb
 - spec/integration/outputs/kafka_spec.rb
@@ -271,6 +287,8 @@ specification_version: 4
 summary: Integration with Kafka - input and output plugins
 test_files:
 - spec/check_docs_spec.rb
+- spec/fixtures/jaas.config
+- spec/fixtures/pwd
 - spec/fixtures/trust-store_stub.jks
 - spec/integration/inputs/kafka_spec.rb
 - spec/integration/outputs/kafka_spec.rb