logstash-integration-kafka 10.7.6-java → 10.7.7-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fc9c66beb8b3a0074d0fa5090dc8896eedd796428e4b7644d8d96e159c7bed2f
-  data.tar.gz: 780f7aeac690ee7a23e954e94c2359d8e9527bcd1464dde007ce7e55c4a023f8
+  metadata.gz: 7a9e872c7a49cd1eb9fc9709c10be72d1c9609b0f19ef5550a2dd7cb317a925e
+  data.tar.gz: b0541ab279e7b7fcea74f1a951e7916abf54adde8bae60e2c4b5897b6f4daadb
 SHA512:
-  metadata.gz: db2e1a6eefc076887de8b42944fe5e0d4ddf025915faf55a4f150710772b08c170e0b4bd629871ca89f4b45fa36b572eb15191c284dec5ad62409c228343e9c5
-  data.tar.gz: b1ae9060e3cf83f6b7c6ca8d949ab2f78696b7a983c7154d8db01e2004875ece63f202ac5778095112adf9597b2d54cedac0ed79682b59bb0bd1b9002265fd36
+  metadata.gz: 1e53d939b471032107afaeb2b7656bf6c755d803e1effbda487b953ae6cc0b79fd97458834d079168023e0ecfa8aeec67a474dc1a4a99336a7be10af3d1b6412
+  data.tar.gz: bea64e388923bab253278e46eccc0701c2bda8f3067cc1e429223fbc3f9cd3a797ced2e334755bbbe3bbf1cf711237359d85e1c6ec22cde822ced2285840fd9f

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 10.7.7
+  - Fix: Correct the settings to allow basic auth to work properly, either by setting `schema_registry_key/secret` or embedding username/password in the
+    url [#94](https://github.com/logstash-plugins/logstash-integration-kafka/pull/94)
+
 ## 10.7.6
   - Test: specify development dependency version [#91](https://github.com/logstash-plugins/logstash-integration-kafka/pull/91)
 

data/lib/logstash/inputs/kafka.rb

@@ -433,13 +433,16 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
       if schema_registry_url
         props.put(kafka::VALUE_DESERIALIZER_CLASS_CONFIG, Java::io.confluent.kafka.serializers.KafkaAvroDeserializer.java_class)
         serdes_config = Java::io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig
-        props.put(serdes_config::SCHEMA_REGISTRY_URL_CONFIG, schema_registry_url.to_s)
+        props.put(serdes_config::SCHEMA_REGISTRY_URL_CONFIG, schema_registry_url.uri.to_s)
         if schema_registry_proxy && !schema_registry_proxy.empty?
           props.put(serdes_config::PROXY_HOST, @schema_registry_proxy_host)
           props.put(serdes_config::PROXY_PORT, @schema_registry_proxy_port)
         end
         if schema_registry_key && !schema_registry_key.empty?
+          props.put(serdes_config::BASIC_AUTH_CREDENTIALS_SOURCE, 'USER_INFO')
           props.put(serdes_config::USER_INFO_CONFIG, schema_registry_key + ":" + schema_registry_secret.value)
+        else
+          props.put(serdes_config::BASIC_AUTH_CREDENTIALS_SOURCE, 'URL')
         end
       end
       if security_protocol == "SSL"

data/lib/logstash/plugin_mixins/common.rb

@@ -53,9 +53,8 @@ module LogStash
           options[:auth] = {:user => schema_registry_key, :password => schema_registry_secret.value}
         end
         client = Manticore::Client.new(options)
-
         begin
-          response = client.get(@schema_registry_url.to_s + '/subjects').body
+          response = client.get(@schema_registry_url.uri.to_s + '/subjects').body
         rescue Manticore::ManticoreException => e
           raise LogStash::ConfigurationError.new("Schema registry service doesn't respond, error: #{e.message}")
         end

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '10.7.6'
+  s.version         = '10.7.7'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+

data/spec/fixtures/jaas.config

@@ -0,0 +1,5 @@
+SchemaRegistry-Props {
+  org.eclipse.jetty.jaas.spi.PropertyFileLoginModule required
+  file="build/confluent_platform/etc/schema-registry/pwd"
+  debug="true";
+};

data/spec/fixtures/pwd

@@ -0,0 +1,5 @@
+fred: OBF:1w8t1tvf1w261w8v1w1c1tvn1w8x,user,admin
+barney: changeme,user,developer
+admin:admin,admin
+betty: MD5:164c88b302622e17050af52c89945d44,user
+wilma: CRYPT:adpexzg3FUZAk,admin,sr-user

data/spec/integration/inputs/kafka_spec.rb

@@ -206,6 +206,16 @@ end
 
 
 describe "schema registry connection options" do
+  schema_registry = Manticore::Client.new
+  before (:all) do
+    shutdown_schema_registry
+    startup_schema_registry(schema_registry)
+  end
+
+  after(:all) do
+    shutdown_schema_registry
+  end
+
   context "remote endpoint validation" do
     it "should fail if not reachable" do
       config = {'schema_registry_url' => 'http://localnothost:8081'}
@@ -232,8 +242,7 @@ describe "schema registry connection options" do
       end
 
       after(:each) do
-        schema_registry_client = Manticore::Client.new
-        delete_remote_schema(schema_registry_client, SUBJECT_NAME)
+        delete_remote_schema(schema_registry, SUBJECT_NAME)
       end
 
       it "should correctly complete registration phase" do
@@ -264,9 +273,25 @@ end
 
 # AdminClientConfig = org.alpache.kafka.clients.admin.AdminClientConfig
 
+def startup_schema_registry(schema_registry, auth=false)
+  system('./stop_schema_registry.sh')
+  auth ? system('./start_auth_schema_registry.sh') : system('./start_schema_registry.sh')
+  url = auth ? "http://barney:changeme@localhost:8081" : "http://localhost:8081"
+  Stud.try(20.times, [Manticore::SocketException, StandardError, RSpec::Expectations::ExpectationNotMetError]) do
+    expect(schema_registry.get(url).code).to eq(200)
+  end
+end
+
 describe "Schema registry API", :integration => true do
+  schema_registry = Manticore::Client.new
+
+  before(:all) do
+    startup_schema_registry(schema_registry)
+  end
 
-  let(:schema_registry) { Manticore::Client.new }
+  after(:all) do
+    shutdown_schema_registry
+  end
 
   context 'listing subject on clean instance' do
     it "should return an empty set" do
@@ -292,37 +317,58 @@ describe "Schema registry API", :integration => true do
       expect( subjects ).to be_empty
     end
   end
+end
+
+def shutdown_schema_registry
+  system('./stop_schema_registry.sh')
+end
+
+describe "Deserializing with the schema registry", :integration => true do
+  schema_registry = Manticore::Client.new
+
+  shared_examples 'it reads from a topic using a schema registry' do |with_auth|
+
+    before(:all) do
+      shutdown_schema_registry
+      startup_schema_registry(schema_registry, with_auth)
+    end
+
+    after(:all) do
+      shutdown_schema_registry
+    end
 
-  context 'use the schema to serialize' do
     after(:each) do
-      expect( schema_registry.delete('http://localhost:8081/subjects/topic_avro-value').code ).to be(200)
+      expect( schema_registry.delete("#{subject_url}/#{avro_topic_name}-value").code ).to be(200)
       sleep 1
-      expect( schema_registry.delete('http://localhost:8081/subjects/topic_avro-value?permanent=true').code ).to be(200)
+      expect( schema_registry.delete("#{subject_url}/#{avro_topic_name}-value?permanent=true").code ).to be(200)
 
       Stud.try(3.times, [StandardError, RSpec::Expectations::ExpectationNotMetError]) do
         wait(10).for do
-          subjects = JSON.parse schema_registry.get('http://localhost:8081/subjects').body
+          subjects = JSON.parse schema_registry.get(subject_url).body
           subjects.empty?
         end.to be_truthy
       end
     end
 
-    let(:group_id_1) {rand(36**8).to_s(36)}
-
-    let(:avro_topic_name) { "topic_avro" }
-
-    let(:plain_config) do
-      { 'schema_registry_url' => 'http://localhost:8081',
-        'topics' => [avro_topic_name],
-        'codec' => 'plain',
-        'group_id' => group_id_1,
-        'auto_offset_reset' => 'earliest' }
+    let(:base_config) do
+      {
+          'topics' => [avro_topic_name],
+          'codec' => 'plain',
+          'group_id' => group_id_1,
+          'auto_offset_reset' => 'earliest'
+      }
     end
 
-    def delete_topic_if_exists(topic_name)
+    let(:group_id_1) {rand(36**8).to_s(36)}
+
+    def delete_topic_if_exists(topic_name, user = nil, password = nil)
       props = java.util.Properties.new
       props.put(Java::org.apache.kafka.clients.admin.AdminClientConfig::BOOTSTRAP_SERVERS_CONFIG, "localhost:9092")
-
+      serdes_config = Java::io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig
+      unless user.nil?
+        props.put(serdes_config::BASIC_AUTH_CREDENTIALS_SOURCE, 'USER_INFO')
+        props.put(serdes_config::USER_INFO_CONFIG,  "#{user}:#{password}")
+      end
       admin_client = org.apache.kafka.clients.admin.AdminClient.create(props)
       topics_list = admin_client.listTopics().names().get()
       if topics_list.contains(topic_name)
@@ -331,7 +377,7 @@ describe "Schema registry API", :integration => true do
       end
     end
 
-    def write_some_data_to(topic_name)
+    def write_some_data_to(topic_name, user = nil, password = nil)
       props = java.util.Properties.new
       config = org.apache.kafka.clients.producer.ProducerConfig
 
@@ -339,6 +385,10 @@ describe "Schema registry API", :integration => true do
       props.put(serdes_config::SCHEMA_REGISTRY_URL_CONFIG, "http://localhost:8081")
 
       props.put(config::BOOTSTRAP_SERVERS_CONFIG, "localhost:9092")
+      unless user.nil?
+        props.put(serdes_config::BASIC_AUTH_CREDENTIALS_SOURCE, 'USER_INFO')
+        props.put(serdes_config::USER_INFO_CONFIG,  "#{user}:#{password}")
+      end
       props.put(config::KEY_SERIALIZER_CLASS_CONFIG, org.apache.kafka.common.serialization.StringSerializer.java_class)
       props.put(config::VALUE_SERIALIZER_CLASS_CONFIG, Java::io.confluent.kafka.serializers.KafkaAvroSerializer.java_class)
 
@@ -360,11 +410,11 @@ describe "Schema registry API", :integration => true do
     end
 
     it "stored a new schema using Avro Kafka serdes" do
-      delete_topic_if_exists avro_topic_name
-      write_some_data_to avro_topic_name
+      auth ? delete_topic_if_exists(avro_topic_name, user, password) : delete_topic_if_exists(avro_topic_name)
+      auth ? write_some_data_to(avro_topic_name, user, password) : write_some_data_to(avro_topic_name)
 
-      subjects = JSON.parse schema_registry.get('http://localhost:8081/subjects').body
-      expect( subjects ).to contain_exactly("topic_avro-value")
+      subjects = JSON.parse schema_registry.get(subject_url).body
+      expect( subjects ).to contain_exactly("#{avro_topic_name}-value")
 
       num_events = 1
       queue = consume_messages(plain_config, timeout: 30, event_count: num_events)
@@ -375,4 +425,43 @@ describe "Schema registry API", :integration => true do
       expect( elem.get("map_field")["inner_field"] ).to eq("inner value")
     end
   end
+
+  context 'with an unauthed schema registry' do
+    let(:auth) { false }
+    let(:avro_topic_name) { "topic_avro" }
+    let(:subject_url) { "http://localhost:8081/subjects" }
+    let(:plain_config)  { base_config.merge!({'schema_registry_url' => "http://localhost:8081"}) }
+
+    it_behaves_like 'it reads from a topic using a schema registry', false
+  end
+
+  context 'with an authed schema registry' do
+    let(:auth) { true }
+    let(:user) { "barney" }
+    let(:password) { "changeme" }
+    let(:avro_topic_name) { "topic_avro_auth" }
+    let(:subject_url) { "http://#{user}:#{password}@localhost:8081/subjects" }
+
+    context 'using schema_registry_key' do
+      let(:plain_config) do
+        base_config.merge!({
+          'schema_registry_url' => "http://localhost:8081",
+          'schema_registry_key' => user,
+          'schema_registry_secret' => password
+        })
+      end
+
+      it_behaves_like 'it reads from a topic using a schema registry', true
+    end
+
+    context 'using schema_registry_url' do
+      let(:plain_config) do
+        base_config.merge!({
+          'schema_registry_url' => "http://#{user}:#{password}@localhost:8081"
+        })
+      end
+
+      it_behaves_like 'it reads from a topic using a schema registry', true
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-kafka
 version: !ruby/object:Gem::Version
-  version: 10.7.6
+  version: 10.7.7
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-05-31 00:00:00.000000000 Z
+date: 2021-07-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -235,6 +235,8 @@ files:
 - lib/logstash/plugin_mixins/kafka_support.rb
 - logstash-integration-kafka.gemspec
 - spec/check_docs_spec.rb
+- spec/fixtures/jaas.config
+- spec/fixtures/pwd
 - spec/fixtures/trust-store_stub.jks
 - spec/integration/inputs/kafka_spec.rb
 - spec/integration/outputs/kafka_spec.rb
@@ -285,6 +287,8 @@ specification_version: 4
 summary: Integration with Kafka - input and output plugins
 test_files:
 - spec/check_docs_spec.rb
+- spec/fixtures/jaas.config
+- spec/fixtures/pwd
 - spec/fixtures/trust-store_stub.jks
 - spec/integration/inputs/kafka_spec.rb
 - spec/integration/outputs/kafka_spec.rb