logstash-integration-kafka 10.5.3-java → 10.6.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4383db6ec7c8fa26ef358d104c490f51620f615afb2f68359b6f6e98d4e58f8b
-  data.tar.gz: 040637202d15cb1e5784104ff505f10a6610e91187a57f104a8f81cd2b24475a
+  metadata.gz: 1d8b40d779e91e9c05dece65249660ab5c272b6833658cbca51b977d92936f42
+  data.tar.gz: 58323e216be645aede9f0b49c27824958b6627485bdf79a6774cd5f87b818245
 SHA512:
-  metadata.gz: 98da085bceebd241a6d45f9166aa4ff1a132551cd2cda8825ceab3c22ebbb5f78579f0d1a0b596aeaf3d504147d656cf2ef784570ef3fd9ab98c792ae6e15be4
-  data.tar.gz: f552e5ec8d84f3ae7d85b3d4bc4ba4f7309321ea81efaa9bfb69a4a493141868b2576a626f0ee061bb0ebe6cbc8071993dd8592bc53030478baad2b5173e1086
+  metadata.gz: 0f05eec028758745a2ab04b90d721128c088c46d0bb9a01923c389118c99a718a561d2ca2420fc2e206e4cf75e4e3695e1704a17d9eff9f4f18e66da3d3ccb85
+  data.tar.gz: 85e117a64d14d013674869ccadfb5487a04991ec83c6a9d6874496b862be200bf4e7203d3ac4fc779d67131ecbe82212fafd66884114337d17ed33dda7ad0963

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 10.6.0
+  - Added functionality to Kafka input to use Avro deserializer in retrieving data from Kafka. The schema is retrieved
+    from an instance of Confluent's Schema Registry service [#51](https://github.com/logstash-plugins/logstash-integration-kafka/pull/51)
+     
 ## 10.5.3
   - Fix: set (optional) truststore when endpoint id check disabled [#60](https://github.com/logstash-plugins/logstash-integration-kafka/pull/60).
     Since **10.1.0** disabling server host-name verification (`ssl_endpoint_identification_algorithm => ""`) did not allow 

data/docs/input-kafka.asciidoc

@@ -124,6 +124,10 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai
 | <<plugins-{type}s-{plugin}-sasl_jaas_config>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-sasl_mechanism>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-schema_registry_key>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-schema_registry_proxy>> |<<uri,uri>>|No
+| <<plugins-{type}s-{plugin}-schema_registry_secret>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-schema_registry_url>> |<<uri,uri>>|No
 | <<plugins-{type}s-{plugin}-security_protocol>> |<<string,string>>, one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No
 | <<plugins-{type}s-{plugin}-send_buffer_bytes>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-session_timeout_ms>> |<<number,number>>|No
@@ -528,6 +532,44 @@ http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used fo
 This may be any mechanism for which a security provider is available.
 GSSAPI is the default mechanism.
 
+[id="plugins-{type}s-{plugin}-schema_registry_key"]
+===== `schema_registry_key`
+
+* Value type is <<string,string>>
+* There is no default value for this setting.
+
+Set the username for basic authorization to access remote Schema Registry.
+
+[id="plugins-{type}s-{plugin}-schema_registry_proxy"]
+===== `schema_registry_proxy`
+
+* Value type is <<uri,uri>>
+* There is no default value for this setting.
+
+Set the address of a forward HTTP proxy. An empty string is treated as if proxy was not set.
+
+[id="plugins-{type}s-{plugin}-schema_registry_secret"]
+===== `schema_registry_secret`
+
+* Value type is <<string,string>>
+* There is no default value for this setting.
+
+Set the password for basic authorization to access remote Schema Registry.
+
+[id="plugins-{type}s-{plugin}-schema_registry_url"]
+===== `schema_registry_url`
+
+* Value type is <<uri,uri>>
+
+The URI that points to an instance of the
+https://docs.confluent.io/current/schema-registry/index.html[Schema Registry] service,
+used to manage Avro schemas. Be sure that the Avro schemas for deserializing the data from
+the specified topics have been uploaded to the Schema Registry service.
+The schemas must follow a naming convention with the pattern <topic name>-value.
+
+Use either the Schema Registry config option or the
+<<plugins-{type}s-{plugin}-value_deserializer_class>> config option, but not both.
+
 [id="plugins-{type}s-{plugin}-security_protocol"]
 ===== `security_protocol` 
 
@@ -641,7 +683,10 @@ The topics configuration will be ignored when using this configuration.
   * Value type is <<string,string>>
   * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"`
 
-Java Class used to deserialize the record's value
+Java Class used to deserialize the record's value.
+A custom value deserializer can be used only if you are not using a Schema Registry.
+Use either the value_deserializer_class config option or the
+<<plugins-{type}s-{plugin}-schema_registry_url>> config option, but not both.
 
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]

data/lib/logstash-integration-kafka_jars.rb

@@ -1,8 +1,17 @@
 # AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
 
 require 'jar_dependencies'
-require_jar('org.apache.kafka', 'kafka-clients', '2.4.1')
-require_jar('com.github.luben', 'zstd-jni', '1.4.3-1')
-require_jar('org.slf4j', 'slf4j-api', '1.7.28')
-require_jar('org.lz4', 'lz4-java', '1.6.0')
+require_jar('io.confluent', 'kafka-avro-serializer', '5.5.1')
+require_jar('io.confluent', 'kafka-schema-serializer', '5.5.1')
+require_jar('io.confluent', 'common-config', '5.5.1')
+require_jar('org.apache.avro', 'avro', '1.9.2')
+require_jar('io.confluent', 'kafka-schema-registry-client', '5.5.1')
+require_jar('org.apache.kafka', 'kafka_2.12', '2.5.1')
+require_jar('io.confluent', 'common-utils', '5.5.1')
+require_jar('javax.ws.rs', 'javax.ws.rs-api', '2.1.1')
+require_jar('org.glassfish.jersey.core', 'jersey-common', '2.30')
+require_jar('org.apache.kafka', 'kafka-clients', '2.5.1')
+require_jar('com.github.luben', 'zstd-jni', '1.4.4-7')
+require_jar('org.slf4j', 'slf4j-api', '1.7.30')
+require_jar('org.lz4', 'lz4-java', '1.7.1')
 require_jar('org.xerial.snappy', 'snappy-java', '1.1.7.3')

data/lib/logstash/inputs/kafka.rb

@@ -4,6 +4,10 @@ require 'stud/interval'
 require 'java'
 require 'logstash-integration-kafka_jars.rb'
 require 'logstash/plugin_mixins/kafka_support'
+require "faraday"
+require "json"
+require "logstash/json"
+require_relative '../plugin_mixins/common'
 
 # This input will read events from a Kafka topic. It uses the 0.10 version of
 # the consumer API provided by Kafka to read messages from the broker.
@@ -50,7 +54,10 @@ require 'logstash/plugin_mixins/kafka_support'
 #
 class LogStash::Inputs::Kafka < LogStash::Inputs::Base
 
+  DEFAULT_DESERIALIZER_CLASS = "org.apache.kafka.common.serialization.StringDeserializer"
+
   include LogStash::PluginMixins::KafkaSupport
+  include ::LogStash::PluginMixins::KafkaAvroSchemaRegistry
 
   config_name 'kafka'
 
@@ -167,7 +174,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   # and a rebalance operation is triggered for the group identified by `group_id`
   config :session_timeout_ms, :validate => :number, :default => 10_000 # (10s) Kafka default
   # Java Class used to deserialize the record's value
-  config :value_deserializer_class, :validate => :string, :default => "org.apache.kafka.common.serialization.StringDeserializer"
+  config :value_deserializer_class, :validate => :string, :default => DEFAULT_DESERIALIZER_CLASS
   # A list of topics to subscribe to, defaults to ["logstash"].
   config :topics, :validate => :array, :default => ["logstash"]
   # A topic regex pattern to subscribe to. 
@@ -236,11 +243,11 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   #   `timestamp`: The timestamp of this message
   config :decorate_events, :validate => :boolean, :default => false
 
-
   public
   def register
     @runner_threads = []
-  end # def register
+    check_schema_registry_parameters
+  end
 
   public
   def run(logstash_queue)
@@ -278,6 +285,13 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
           for record in records do
             codec_instance.decode(record.value.to_s) do |event|
               decorate(event)
+              if schema_registry_url
+                json = LogStash::Json.load(record.value.to_s)
+                json.each do |k, v|
+                  event.set(k, v)
+                end
+                event.remove("message")
+              end
               if @decorate_events
                 event.set("[@metadata][kafka][topic]", record.topic)
                 event.set("[@metadata][kafka][consumer_group]", @group_id)
@@ -337,7 +351,18 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
       props.put(kafka::CLIENT_RACK_CONFIG, client_rack) unless client_rack.nil? 
 
       props.put("security.protocol", security_protocol) unless security_protocol.nil?
-
+      if schema_registry_url
+        props.put(kafka::VALUE_DESERIALIZER_CLASS_CONFIG, Java::io.confluent.kafka.serializers.KafkaAvroDeserializer.java_class)
+        serdes_config = Java::io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig
+        props.put(serdes_config::SCHEMA_REGISTRY_URL_CONFIG, schema_registry_url.to_s)
+        if schema_registry_proxy && !schema_registry_proxy.empty?
+          props.put(serdes_config::PROXY_HOST, @schema_registry_proxy_host)
+          props.put(serdes_config::PROXY_PORT, @schema_registry_proxy_port)
+        end
+        if schema_registry_key && !schema_registry_key.empty?
+          props.put(serdes_config::USER_INFO_CONFIG, schema_registry_key + ":" + schema_registry_secret.value)
+        end
+      end
       if security_protocol == "SSL"
         set_trustore_keystore_config(props)
       elsif security_protocol == "SASL_PLAINTEXT"

data/lib/logstash/plugin_mixins/common.rb

@@ -0,0 +1,92 @@
+module LogStash
+  module PluginMixins
+    module KafkaAvroSchemaRegistry
+
+      def self.included(base)
+        base.extend(self)
+        base.setup_schema_registry_config
+      end
+
+      def setup_schema_registry_config
+        # Option to set key to access Schema Registry.
+        config :schema_registry_key, :validate => :string
+
+        # Option to set secret to access Schema Registry.
+        config :schema_registry_secret, :validate => :password
+
+        # Option to set the endpoint of the Schema Registry.
+        # This option permit the usage of Avro Kafka deserializer which retrieve the schema of the Avro message from an
+        # instance of schema registry. If this option has value `value_deserializer_class` nor `topics_pattern` could be valued
+        config :schema_registry_url, :validate => :uri
+
+        # Option to set the proxy of the Schema Registry.
+        # This option permits to define a proxy to be used to reach the schema registry service instance.
+        config :schema_registry_proxy, :validate => :uri
+      end
+
+      def check_schema_registry_parameters
+        if @schema_registry_url
+          check_for_schema_registry_conflicts
+          @schema_registry_proxy_host, @schema_registry_proxy_port  = split_proxy_into_host_and_port(schema_registry_proxy)
+          check_for_key_and_secret
+          check_for_schema_registry_connectivity_and_subjects
+        end
+      end
+
+      private
+      def check_for_schema_registry_conflicts
+        if @value_deserializer_class != LogStash::Inputs::Kafka::DEFAULT_DESERIALIZER_CLASS
+          raise LogStash::ConfigurationError, 'Option schema_registry_url prohibit the customization of value_deserializer_class'
+        end
+        if @topics_pattern && !@topics_pattern.empty?
+         raise LogStash::ConfigurationError, 'Option schema_registry_url prohibit the customization of topics_pattern'
+        end
+      end
+
+      private
+      def check_for_schema_registry_connectivity_and_subjects
+        client = Faraday.new(@schema_registry_url.to_s) do |conn|
+          if schema_registry_proxy && !schema_registry_proxy.empty?
+            conn.proxy = schema_registry_proxy.to_s
+          end
+          if schema_registry_key and !schema_registry_key.empty?
+            conn.basic_auth(schema_registry_key, schema_registry_secret.value)
+          end
+        end
+        begin
+          response = client.get('/subjects')
+        rescue Faraday::Error => e
+          raise LogStash::ConfigurationError.new("Schema registry service doesn't respond, error: #{e.message}")
+        end
+        registered_subjects = JSON.parse response.body
+        expected_subjects = @topics.map { |t| "#{t}-value"}
+        if (expected_subjects & registered_subjects).size != expected_subjects.size
+          undefined_topic_subjects = expected_subjects - registered_subjects
+          raise LogStash::ConfigurationError, "The schema registry does not contain definitions for required topic subjects: #{undefined_topic_subjects}"
+        end
+      end
+
+      def split_proxy_into_host_and_port(proxy_uri)
+        return nil unless proxy_uri && !proxy_uri.empty?
+
+        port = proxy_uri.port
+
+        host_spec = ""
+        host_spec << proxy_uri.scheme || "http"
+        host_spec << "://"
+        host_spec << "#{proxy_uri.userinfo}@" if proxy_uri.userinfo
+        host_spec << proxy_uri.host
+
+        [host_spec, port]
+      end
+
+      def check_for_key_and_secret
+        if schema_registry_key and !schema_registry_key.empty?
+          if !schema_registry_secret or schema_registry_secret.value.empty?
+            raise LogStash::ConfigurationError, "Setting `schema_registry_secret` is required when `schema_registry_key` is provided."
+          end
+        end
+      end
+    end
+  end
+end

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '10.5.3'
+  s.version         = '10.6.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+

data/spec/integration/inputs/kafka_spec.rb

@@ -2,6 +2,9 @@
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/inputs/kafka"
 require "rspec/wait"
+require "stud/try"
+require "faraday"
+require "json"
 
 # Please run kafka_test_setup.sh prior to executing this integration test.
 describe "inputs/kafka", :integration => true do
@@ -120,20 +123,192 @@ describe "inputs/kafka", :integration => true do
       end
     end
   end
+end
+
+private
+
+def consume_messages(config, queue: Queue.new, timeout:, event_count:)
+  kafka_input = LogStash::Inputs::Kafka.new(config)
+  t = Thread.new { kafka_input.run(queue) }
+  begin
+    t.run
+    wait(timeout).for { queue.length }.to eq(event_count) unless timeout.eql?(false)
+    block_given? ? yield(queue, kafka_input) : queue
+  ensure
+    t.kill
+    t.join(30_000)
+  end
+end
+
+
+describe "schema registry connection options" do
+  context "remote endpoint validation" do
+    it "should fail if not reachable" do
+      config = {'schema_registry_url' => 'http://localnothost:8081'}
+      kafka_input = LogStash::Inputs::Kafka.new(config)
+      expect { kafka_input.register }.to raise_error LogStash::ConfigurationError, /Schema registry service doesn't respond.*/
+    end
+
+    it "should fail if any topic is not matched by a subject on the schema registry" do
+      config = {
+        'schema_registry_url' => 'http://localhost:8081',
+        'topics' => ['temperature_stream']
+      }
+
+      kafka_input = LogStash::Inputs::Kafka.new(config)
+      expect { kafka_input.register }.to raise_error LogStash::ConfigurationError, /The schema registry does not contain definitions for required topic subjects: \["temperature_stream-value"\]/
+    end
+
+    context "register with subject present" do
+      SUBJECT_NAME = "temperature_stream-value"
+
+      before(:each) do
+        response = save_avro_schema_to_schema_registry(File.join(Dir.pwd, "spec", "unit", "inputs", "avro_schema_fixture_payment.asvc"), SUBJECT_NAME)
+        expect( response.status ).to be(200)
+      end
 
-  private
+      after(:each) do
+        schema_registry_client = Faraday.new('http://localhost:8081')
+        delete_remote_schema(schema_registry_client, SUBJECT_NAME)
+      end
 
-  def consume_messages(config, queue: Queue.new, timeout:, event_count:)
-    kafka_input = LogStash::Inputs::Kafka.new(config)
-    t = Thread.new { kafka_input.run(queue) }
-    begin
-      t.run
-      wait(timeout).for { queue.length }.to eq(event_count) unless timeout.eql?(false)
-      block_given? ? yield(queue, kafka_input) : queue
-    ensure
-      t.kill
-      t.join(30_000)
+      it "should correctly complete registration phase" do
+        config = {
+          'schema_registry_url' => 'http://localhost:8081',
+          'topics' => ['temperature_stream']
+        }
+        kafka_input = LogStash::Inputs::Kafka.new(config)
+        kafka_input.register
+      end
     end
   end
+end
 
+def save_avro_schema_to_schema_registry(schema_file, subject_name)
+  raw_schema = File.readlines(schema_file).map(&:chomp).join
+  raw_schema_quoted = raw_schema.gsub('"', '\"')
+  response = Faraday.post("http://localhost:8081/subjects/#{subject_name}/versions",
+          '{"schema": "' + raw_schema_quoted + '"}',
+          "Content-Type" => "application/vnd.schemaregistry.v1+json")
+  response
 end
+
+def delete_remote_schema(schema_registry_client, subject_name)
+  expect(schema_registry_client.delete("/subjects/#{subject_name}").status ).to be(200)
+  expect(schema_registry_client.delete("/subjects/#{subject_name}?permanent=true").status ).to be(200)
+end
+
+# AdminClientConfig = org.alpache.kafka.clients.admin.AdminClientConfig
+
+describe "Schema registry API", :integration => true do
+
+  let(:schema_registry) { Faraday.new('http://localhost:8081') }
+
+  context 'listing subject on clean instance' do
+    it "should return an empty set" do
+      subjects = JSON.parse schema_registry.get('/subjects').body
+      expect( subjects ).to be_empty
+    end
+  end
+
+  context 'send a schema definition' do
+    it "save the definition" do
+      response = save_avro_schema_to_schema_registry(File.join(Dir.pwd, "spec", "unit", "inputs", "avro_schema_fixture_payment.asvc"), "schema_test_1")
+      expect( response.status ).to be(200)
+      delete_remote_schema(schema_registry, "schema_test_1")
+    end
+
+    it "delete the schema just added" do
+      response = save_avro_schema_to_schema_registry(File.join(Dir.pwd, "spec", "unit", "inputs", "avro_schema_fixture_payment.asvc"), "schema_test_1")
+      expect( response.status ).to be(200)
+
+      expect( schema_registry.delete('/subjects/schema_test_1?permanent=false').status ).to be(200)
+      sleep(1)
+      subjects = JSON.parse schema_registry.get('/subjects').body
+      expect( subjects ).to be_empty
+    end
+  end
+
+  context 'use the schema to serialize' do
+    after(:each) do
+      expect( schema_registry.delete('/subjects/topic_avro-value').status ).to be(200)
+      sleep 1
+      expect( schema_registry.delete('/subjects/topic_avro-value?permanent=true').status ).to be(200)
+
+      Stud.try(3.times, [StandardError, RSpec::Expectations::ExpectationNotMetError]) do
+        wait(10).for do
+          subjects = JSON.parse schema_registry.get('/subjects').body
+          subjects.empty?
+        end.to be_truthy
+      end
+    end
+
+    let(:group_id_1) {rand(36**8).to_s(36)}
+
+    let(:avro_topic_name) { "topic_avro" }
+
+    let(:plain_config) do
+      { 'schema_registry_url' => 'http://localhost:8081',
+        'topics' => [avro_topic_name],
+        'codec' => 'plain',
+        'group_id' => group_id_1,
+        'auto_offset_reset' => 'earliest' }
+    end
+
+    def delete_topic_if_exists(topic_name)
+      props = java.util.Properties.new
+      props.put(Java::org.apache.kafka.clients.admin.AdminClientConfig::BOOTSTRAP_SERVERS_CONFIG, "localhost:9092")
+
+      admin_client = org.apache.kafka.clients.admin.AdminClient.create(props)
+      topics_list = admin_client.listTopics().names().get()
+      if topics_list.contains(topic_name)
+        result = admin_client.deleteTopics([topic_name])
+        result.values.get(topic_name).get()
+      end
+    end
+
+    def write_some_data_to(topic_name)
+      props = java.util.Properties.new
+      config = org.apache.kafka.clients.producer.ProducerConfig
+
+      serdes_config = Java::io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig
+      props.put(serdes_config::SCHEMA_REGISTRY_URL_CONFIG, "http://localhost:8081")
+
+      props.put(config::BOOTSTRAP_SERVERS_CONFIG, "localhost:9092")
+      props.put(config::KEY_SERIALIZER_CLASS_CONFIG, org.apache.kafka.common.serialization.StringSerializer.java_class)
+      props.put(config::VALUE_SERIALIZER_CLASS_CONFIG, Java::io.confluent.kafka.serializers.KafkaAvroSerializer.java_class)
+
+      parser = org.apache.avro.Schema::Parser.new()
+      user_schema = '''{"type":"record",
+                        "name":"myrecord",
+                        "fields":[
+                            {"name":"str_field", "type": "string"},
+                            {"name":"map_field", "type": {"type": "map", "values": "string"}}
+                        ]}'''
+      schema = parser.parse(user_schema)
+      avro_record = org.apache.avro.generic.GenericData::Record.new(schema)
+      avro_record.put("str_field", "value1")
+      avro_record.put("map_field", {"inner_field" => "inner value"})
+
+      producer = org.apache.kafka.clients.producer.KafkaProducer.new(props)
+      record = org.apache.kafka.clients.producer.ProducerRecord.new(topic_name, "avro_key", avro_record)
+      producer.send(record)
+    end
+
+    it "stored a new schema using Avro Kafka serdes" do
+      delete_topic_if_exists avro_topic_name
+      write_some_data_to avro_topic_name
+
+      subjects = JSON.parse schema_registry.get('/subjects').body
+      expect( subjects ).to contain_exactly("topic_avro-value")
+
+      num_events = 1
+      queue = consume_messages(plain_config, timeout: 30, event_count: num_events)
+      expect(queue.length).to eq(num_events)
+      elem = queue.pop
+      expect( elem.to_hash).not_to include("message")
+      expect( elem.get("str_field") ).to eq("value1")
+      expect( elem.get("map_field")["inner_field"] ).to eq("inner value")
+    end
+  end
+end

data/spec/unit/inputs/avro_schema_fixture_payment.asvc

@@ -0,0 +1,8 @@
+{"namespace": "io.confluent.examples.clients.basicavro",
+ "type": "record",
+ "name": "Payment",
+ "fields": [
+     {"name": "id", "type": "string"},
+     {"name": "amount", "type": "double"}
+ ]
+}

data/spec/unit/inputs/kafka_spec.rb

@@ -37,6 +37,22 @@ describe LogStash::Inputs::Kafka do
     expect { subject.register }.to_not raise_error
   end
 
+  context "register parameter verification" do
+    let(:config) do
+      { 'schema_registry_url' => 'http://localhost:8081', 'topics' => ['logstash'], 'consumer_threads' => 4 }
+    end
+
+    it "schema_registry_url conflict with value_deserializer_class should fail" do
+      config['value_deserializer_class'] = 'my.fantasy.Deserializer'
+      expect { subject.register }.to raise_error LogStash::ConfigurationError, /Option schema_registry_url prohibit the customization of value_deserializer_class/
+    end
+
+    it "schema_registry_url conflict with topics_pattern should fail" do
+      config['topics_pattern'] = 'topic_.*'
+      expect { subject.register }.to raise_error LogStash::ConfigurationError, /Option schema_registry_url prohibit the customization of topics_pattern/
+    end
+  end
+
   context 'with client_rack' do
     let(:config) { super.merge('client_rack' => 'EU-R1') }
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-kafka
 version: !ruby/object:Gem::Version
-  version: 10.5.3
+  version: 10.6.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-21 00:00:00.000000000 Z
+date: 2020-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -183,17 +183,28 @@ files:
 - lib/logstash-integration-kafka_jars.rb
 - lib/logstash/inputs/kafka.rb
 - lib/logstash/outputs/kafka.rb
+- lib/logstash/plugin_mixins/common.rb
 - lib/logstash/plugin_mixins/kafka_support.rb
 - logstash-integration-kafka.gemspec
 - spec/fixtures/trust-store_stub.jks
 - spec/integration/inputs/kafka_spec.rb
 - spec/integration/outputs/kafka_spec.rb
+- spec/unit/inputs/avro_schema_fixture_payment.asvc
 - spec/unit/inputs/kafka_spec.rb
 - spec/unit/outputs/kafka_spec.rb
-- vendor/jar-dependencies/com/github/luben/zstd-jni/1.4.3-1/zstd-jni-1.4.3-1.jar
-- vendor/jar-dependencies/org/apache/kafka/kafka-clients/2.4.1/kafka-clients-2.4.1.jar
-- vendor/jar-dependencies/org/lz4/lz4-java/1.6.0/lz4-java-1.6.0.jar
-- vendor/jar-dependencies/org/slf4j/slf4j-api/1.7.28/slf4j-api-1.7.28.jar
+- vendor/jar-dependencies/com/github/luben/zstd-jni/1.4.4-7/zstd-jni-1.4.4-7.jar
+- vendor/jar-dependencies/io/confluent/common-config/5.5.1/common-config-5.5.1.jar
+- vendor/jar-dependencies/io/confluent/common-utils/5.5.1/common-utils-5.5.1.jar
+- vendor/jar-dependencies/io/confluent/kafka-avro-serializer/5.5.1/kafka-avro-serializer-5.5.1.jar
+- vendor/jar-dependencies/io/confluent/kafka-schema-registry-client/5.5.1/kafka-schema-registry-client-5.5.1.jar
+- vendor/jar-dependencies/io/confluent/kafka-schema-serializer/5.5.1/kafka-schema-serializer-5.5.1.jar
+- vendor/jar-dependencies/javax/ws/rs/javax.ws.rs-api/2.1.1/javax.ws.rs-api-2.1.1.jar
+- vendor/jar-dependencies/org/apache/avro/avro/1.9.2/avro-1.9.2.jar
+- vendor/jar-dependencies/org/apache/kafka/kafka-clients/2.5.1/kafka-clients-2.5.1.jar
+- vendor/jar-dependencies/org/apache/kafka/kafka_2.12/2.5.1/kafka_2.12-2.5.1.jar
+- vendor/jar-dependencies/org/glassfish/jersey/core/jersey-common/2.30/jersey-common-2.30.jar
+- vendor/jar-dependencies/org/lz4/lz4-java/1.7.1/lz4-java-1.7.1.jar
+- vendor/jar-dependencies/org/slf4j/slf4j-api/1.7.30/slf4j-api-1.7.30.jar
 - vendor/jar-dependencies/org/xerial/snappy/snappy-java/1.1.7.3/snappy-java-1.1.7.3.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
@@ -227,5 +238,6 @@ test_files:
 - spec/fixtures/trust-store_stub.jks
 - spec/integration/inputs/kafka_spec.rb
 - spec/integration/outputs/kafka_spec.rb
+- spec/unit/inputs/avro_schema_fixture_payment.asvc
 - spec/unit/inputs/kafka_spec.rb
 - spec/unit/outputs/kafka_spec.rb