logstash-integration-kafka 10.5.3-java → 10.7.3-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4383db6ec7c8fa26ef358d104c490f51620f615afb2f68359b6f6e98d4e58f8b
-  data.tar.gz: 040637202d15cb1e5784104ff505f10a6610e91187a57f104a8f81cd2b24475a
+  metadata.gz: 1049dfcad573e64aed24bebc6114b0345ce6513e43c2104c54ac8e3ed6602e57
+  data.tar.gz: 41260002d0a2765b4beeac73bdd8d8f100e0b3e058c1e5dbac4f2a460a29a3d0
 SHA512:
-  metadata.gz: 98da085bceebd241a6d45f9166aa4ff1a132551cd2cda8825ceab3c22ebbb5f78579f0d1a0b596aeaf3d504147d656cf2ef784570ef3fd9ab98c792ae6e15be4
-  data.tar.gz: f552e5ec8d84f3ae7d85b3d4bc4ba4f7309321ea81efaa9bfb69a4a493141868b2576a626f0ee061bb0ebe6cbc8071993dd8592bc53030478baad2b5173e1086
+  metadata.gz: 744b4621e51c32ef882c7f5521be467c75805c2ba6ee8eaf225567412fe9e6bade35e60ae7fa85b87de50f68a191f5408b0504025e28071e11eb227ab5f428c8
+  data.tar.gz: feb75d5f0da3523e891253213cf52b49460881c125adf2fc82710f96c0feaa3eb4630468bbec7f1182c6a52e44b060362b5dd946b6194c066821a3de4c6e1314

data/CHANGELOG.md

@@ -1,3 +1,20 @@
+## 10.7.3
+  - Changed `decorate_events` to add also Kafka headers [#78](https://github.com/logstash-plugins/logstash-integration-kafka/pull/78)
+
+## 10.7.2
+  - Update Jersey dependency to version 2.33 [#75](https://github.com/logstash-plugins/logstash-integration-kafka/pull/75)
+
+## 10.7.1
+  - Fix: dropped usage of SHUTDOWN event deprecated since Logstash 5.0 [#71](https://github.com/logstash-plugins/logstash-integration-kafka/pull/71)
+  
+## 10.7.0
+  - Switched use from Faraday to Manticore as HTTP client library to access Schema Registry service 
+    to fix issue [#63](https://github.com/logstash-plugins/logstash-integration-kafka/pull/63) 
+
+## 10.6.0
+  - Added functionality to Kafka input to use Avro deserializer in retrieving data from Kafka. The schema is retrieved
+    from an instance of Confluent's Schema Registry service [#51](https://github.com/logstash-plugins/logstash-integration-kafka/pull/51)
+     
 ## 10.5.3
   - Fix: set (optional) truststore when endpoint id check disabled [#60](https://github.com/logstash-plugins/logstash-integration-kafka/pull/60).
     Since **10.1.0** disabling server host-name verification (`ssl_endpoint_identification_algorithm => ""`) did not allow 

data/README.md

@@ -1,6 +1,6 @@
 # Logstash Plugin
 
-[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-integration-kafka.svg)](https://travis-ci.org/logstash-plugins/logstash-integration-kafka)
+[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-integration-kafka.svg)](https://travis-ci.com/logstash-plugins/logstash-integration-kafka)
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 

data/docs/input-kafka.asciidoc

@@ -73,7 +73,7 @@ either when the record was created (default) or when it was received by the
 broker. See more about property log.message.timestamp.type at
 https://kafka.apache.org/{kafka_client_doc}/documentation.html#brokerconfigs
 
-Metadata is only added to the event if the `decorate_events` option is set to true (it defaults to false).
+Metadata is only added to the event if the `decorate_events` option is set to `basic` or `extended` (it defaults to `none`).
 
 Please note that `@metadata` fields are not part of any of your events at output time. If you need these information to be 
 inserted into your original event, you'll have to use the `mutate` filter to manually copy the required fields into your `event`.
@@ -99,7 +99,7 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai
 | <<plugins-{type}s-{plugin}-client_rack>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-connections_max_idle_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-consumer_threads>> |<<number,number>>|No
-| <<plugins-{type}s-{plugin}-decorate_events>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-decorate_events>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-enable_auto_commit>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-exclude_internal_topics>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-fetch_max_bytes>> |<<number,number>>|No
@@ -124,6 +124,10 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai
 | <<plugins-{type}s-{plugin}-sasl_jaas_config>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-sasl_kerberos_service_name>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-sasl_mechanism>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-schema_registry_key>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-schema_registry_proxy>> |<<uri,uri>>|No
+| <<plugins-{type}s-{plugin}-schema_registry_secret>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-schema_registry_url>> |<<uri,uri>>|No
 | <<plugins-{type}s-{plugin}-security_protocol>> |<<string,string>>, one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No
 | <<plugins-{type}s-{plugin}-send_buffer_bytes>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-session_timeout_ms>> |<<number,number>>|No
@@ -242,10 +246,16 @@ balance — more threads than partitions means that some threads will be idl
 [id="plugins-{type}s-{plugin}-decorate_events"]
 ===== `decorate_events` 
 
-  * Value type is <<boolean,boolean>>
-  * Default value is `false`
-
-Option to add Kafka metadata like topic, message size to the event.
+  * Value type is <<string,string>>
+  * Accepted values are:
+    - `none`: no metadata is added
+    - `basic`: record's attributes are added
+    - `extended`: record's attributes, headers are added
+    - `false`: deprecated alias for `none`
+    - `true`: deprecated alias for `basic`
+  * Default value is `none`
+
+Option to add Kafka metadata like topic, message size and header key values to the event.
 This will add a field named `kafka` to the logstash event containing the following attributes:
 
 * `topic`: The topic this message is associated with
@@ -528,6 +538,44 @@ http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used fo
 This may be any mechanism for which a security provider is available.
 GSSAPI is the default mechanism.
 
+[id="plugins-{type}s-{plugin}-schema_registry_key"]
+===== `schema_registry_key`
+
+* Value type is <<string,string>>
+* There is no default value for this setting.
+
+Set the username for basic authorization to access remote Schema Registry.
+
+[id="plugins-{type}s-{plugin}-schema_registry_proxy"]
+===== `schema_registry_proxy`
+
+* Value type is <<uri,uri>>
+* There is no default value for this setting.
+
+Set the address of a forward HTTP proxy. An empty string is treated as if proxy was not set.
+
+[id="plugins-{type}s-{plugin}-schema_registry_secret"]
+===== `schema_registry_secret`
+
+* Value type is <<string,string>>
+* There is no default value for this setting.
+
+Set the password for basic authorization to access remote Schema Registry.
+
+[id="plugins-{type}s-{plugin}-schema_registry_url"]
+===== `schema_registry_url`
+
+* Value type is <<uri,uri>>
+
+The URI that points to an instance of the
+https://docs.confluent.io/current/schema-registry/index.html[Schema Registry] service,
+used to manage Avro schemas. Be sure that the Avro schemas for deserializing the data from
+the specified topics have been uploaded to the Schema Registry service.
+The schemas must follow a naming convention with the pattern <topic name>-value.
+
+Use either the Schema Registry config option or the
+<<plugins-{type}s-{plugin}-value_deserializer_class>> config option, but not both.
+
 [id="plugins-{type}s-{plugin}-security_protocol"]
 ===== `security_protocol` 
 
@@ -641,7 +689,10 @@ The topics configuration will be ignored when using this configuration.
   * Value type is <<string,string>>
   * Default value is `"org.apache.kafka.common.serialization.StringDeserializer"`
 
-Java Class used to deserialize the record's value
+Java Class used to deserialize the record's value.
+A custom value deserializer can be used only if you are not using a Schema Registry.
+Use either the value_deserializer_class config option or the
+<<plugins-{type}s-{plugin}-schema_registry_url>> config option, but not both.
 
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]

data/lib/logstash-integration-kafka_jars.rb

@@ -1,8 +1,17 @@
 # AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
 
 require 'jar_dependencies'
-require_jar('org.apache.kafka', 'kafka-clients', '2.4.1')
-require_jar('com.github.luben', 'zstd-jni', '1.4.3-1')
-require_jar('org.slf4j', 'slf4j-api', '1.7.28')
-require_jar('org.lz4', 'lz4-java', '1.6.0')
+require_jar('io.confluent', 'kafka-avro-serializer', '5.5.1')
+require_jar('io.confluent', 'kafka-schema-serializer', '5.5.1')
+require_jar('io.confluent', 'common-config', '5.5.1')
+require_jar('org.apache.avro', 'avro', '1.9.2')
+require_jar('io.confluent', 'kafka-schema-registry-client', '5.5.1')
+require_jar('org.apache.kafka', 'kafka_2.12', '2.5.1')
+require_jar('io.confluent', 'common-utils', '5.5.1')
+require_jar('javax.ws.rs', 'javax.ws.rs-api', '2.1.1')
+require_jar('org.glassfish.jersey.core', 'jersey-common', '2.33')
+require_jar('org.apache.kafka', 'kafka-clients', '2.5.1')
+require_jar('com.github.luben', 'zstd-jni', '1.4.4-7')
+require_jar('org.slf4j', 'slf4j-api', '1.7.30')
+require_jar('org.lz4', 'lz4-java', '1.7.1')
 require_jar('org.xerial.snappy', 'snappy-java', '1.1.7.3')

data/lib/logstash/inputs/kafka.rb

@@ -4,6 +4,11 @@ require 'stud/interval'
 require 'java'
 require 'logstash-integration-kafka_jars.rb'
 require 'logstash/plugin_mixins/kafka_support'
+require 'manticore'
+require "json"
+require "logstash/json"
+require_relative '../plugin_mixins/common'
+require 'logstash/plugin_mixins/deprecation_logger_support'
 
 # This input will read events from a Kafka topic. It uses the 0.10 version of
 # the consumer API provided by Kafka to read messages from the broker.
@@ -50,7 +55,11 @@ require 'logstash/plugin_mixins/kafka_support'
 #
 class LogStash::Inputs::Kafka < LogStash::Inputs::Base
 
+  DEFAULT_DESERIALIZER_CLASS = "org.apache.kafka.common.serialization.StringDeserializer"
+
   include LogStash::PluginMixins::KafkaSupport
+  include ::LogStash::PluginMixins::KafkaAvroSchemaRegistry
+  include LogStash::PluginMixins::DeprecationLoggerSupport
 
   config_name 'kafka'
 
@@ -167,7 +176,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   # and a rebalance operation is triggered for the group identified by `group_id`
   config :session_timeout_ms, :validate => :number, :default => 10_000 # (10s) Kafka default
   # Java Class used to deserialize the record's value
-  config :value_deserializer_class, :validate => :string, :default => "org.apache.kafka.common.serialization.StringDeserializer"
+  config :value_deserializer_class, :validate => :string, :default => DEFAULT_DESERIALIZER_CLASS
   # A list of topics to subscribe to, defaults to ["logstash"].
   config :topics, :validate => :array, :default => ["logstash"]
   # A topic regex pattern to subscribe to. 
@@ -226,21 +235,48 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   config :sasl_jaas_config, :validate => :string
   # Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html
   config :kerberos_config, :validate => :path
-  # Option to add Kafka metadata like topic, message size to the event.
-  # This will add a field named `kafka` to the logstash event containing the following attributes:
+  # Option to add Kafka metadata like topic, message size and header key values to the event.
+  # With `basic` this will add a field named `kafka` to the logstash event containing the following attributes:
   #   `topic`: The topic this message is associated with
   #   `consumer_group`: The consumer group used to read in this event
   #   `partition`: The partition this message is associated with
   #   `offset`: The offset from the partition this message is associated with
   #   `key`: A ByteBuffer containing the message key
   #   `timestamp`: The timestamp of this message
-  config :decorate_events, :validate => :boolean, :default => false
+  # While with `extended` it adds also all the key values present in the Kafka header if the key is valid UTF-8 else
+  # silently skip it.
+  config :decorate_events, :validate => %w(none basic extended false true), :default => "none"
 
+  attr_reader :metadata_mode
 
   public
   def register
     @runner_threads = []
-  end # def register
+    @metadata_mode = extract_metadata_level(@decorate_events)
+    check_schema_registry_parameters
+  end
+
+  METADATA_NONE     = Set[].freeze
+  METADATA_BASIC    = Set[:record_props].freeze
+  METADATA_EXTENDED = Set[:record_props, :headers].freeze
+  METADATA_DEPRECATION_MAP = { 'true' => 'basic', 'false' => 'none' }
+
+  private
+  def extract_metadata_level(decorate_events_setting)
+    metadata_enabled = decorate_events_setting
+
+    if METADATA_DEPRECATION_MAP.include?(metadata_enabled)
+      canonical_value = METADATA_DEPRECATION_MAP[metadata_enabled]
+      deprecation_logger.deprecated("Deprecated value `#{decorate_events_setting}` for `decorate_events` option; use `#{canonical_value}` instead.")
+      metadata_enabled = canonical_value
+    end
+
+    case metadata_enabled
+    when 'none'     then METADATA_NONE
+    when 'basic'    then METADATA_BASIC
+    when 'extended' then METADATA_EXTENDED
+    end
+  end
 
   public
   def run(logstash_queue)
@@ -278,7 +314,14 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
           for record in records do
             codec_instance.decode(record.value.to_s) do |event|
               decorate(event)
-              if @decorate_events
+              if schema_registry_url
+                json = LogStash::Json.load(record.value.to_s)
+                json.each do |k, v|
+                  event.set(k, v)
+                end
+                event.remove("message")
+              end
+              if @metadata_mode.include?(:record_props)
                 event.set("[@metadata][kafka][topic]", record.topic)
                 event.set("[@metadata][kafka][consumer_group]", @group_id)
                 event.set("[@metadata][kafka][partition]", record.partition)
@@ -286,6 +329,15 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
                 event.set("[@metadata][kafka][key]", record.key)
                 event.set("[@metadata][kafka][timestamp]", record.timestamp)
               end
+              if @metadata_mode.include?(:headers)
+                record.headers.each do |header|
+                  s = String.from_java_bytes(header.value)
+                  s.force_encoding(Encoding::UTF_8)
+                  if s.valid_encoding?
+                    event.set("[@metadata][kafka][headers]["+header.key+"]", s)
+                  end
+                end
+              end
               logstash_queue << event
             end
           end
@@ -337,7 +389,18 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
       props.put(kafka::CLIENT_RACK_CONFIG, client_rack) unless client_rack.nil? 
 
       props.put("security.protocol", security_protocol) unless security_protocol.nil?
-
+      if schema_registry_url
+        props.put(kafka::VALUE_DESERIALIZER_CLASS_CONFIG, Java::io.confluent.kafka.serializers.KafkaAvroDeserializer.java_class)
+        serdes_config = Java::io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig
+        props.put(serdes_config::SCHEMA_REGISTRY_URL_CONFIG, schema_registry_url.to_s)
+        if schema_registry_proxy && !schema_registry_proxy.empty?
+          props.put(serdes_config::PROXY_HOST, @schema_registry_proxy_host)
+          props.put(serdes_config::PROXY_PORT, @schema_registry_proxy_port)
+        end
+        if schema_registry_key && !schema_registry_key.empty?
+          props.put(serdes_config::USER_INFO_CONFIG, schema_registry_key + ":" + schema_registry_secret.value)
+        end
+      end
       if security_protocol == "SSL"
         set_trustore_keystore_config(props)
       elsif security_protocol == "SASL_PLAINTEXT"

data/lib/logstash/outputs/kafka.rb

@@ -224,7 +224,6 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
     end
 
     events.each do |event|
-      break if event == LogStash::SHUTDOWN
       @codec.encode(event)
     end
 

data/lib/logstash/plugin_mixins/common.rb

@@ -0,0 +1,93 @@
+module LogStash
+  module PluginMixins
+    module KafkaAvroSchemaRegistry
+
+      def self.included(base)
+        base.extend(self)
+        base.setup_schema_registry_config
+      end
+
+      def setup_schema_registry_config
+        # Option to set key to access Schema Registry.
+        config :schema_registry_key, :validate => :string
+
+        # Option to set secret to access Schema Registry.
+        config :schema_registry_secret, :validate => :password
+
+        # Option to set the endpoint of the Schema Registry.
+        # This option permit the usage of Avro Kafka deserializer which retrieve the schema of the Avro message from an
+        # instance of schema registry. If this option has value `value_deserializer_class` nor `topics_pattern` could be valued
+        config :schema_registry_url, :validate => :uri
+
+        # Option to set the proxy of the Schema Registry.
+        # This option permits to define a proxy to be used to reach the schema registry service instance.
+        config :schema_registry_proxy, :validate => :uri
+      end
+
+      def check_schema_registry_parameters
+        if @schema_registry_url
+          check_for_schema_registry_conflicts
+          @schema_registry_proxy_host, @schema_registry_proxy_port  = split_proxy_into_host_and_port(schema_registry_proxy)
+          check_for_key_and_secret
+          check_for_schema_registry_connectivity_and_subjects
+        end
+      end
+
+      private
+      def check_for_schema_registry_conflicts
+        if @value_deserializer_class != LogStash::Inputs::Kafka::DEFAULT_DESERIALIZER_CLASS
+          raise LogStash::ConfigurationError, 'Option schema_registry_url prohibit the customization of value_deserializer_class'
+        end
+        if @topics_pattern && !@topics_pattern.empty?
+         raise LogStash::ConfigurationError, 'Option schema_registry_url prohibit the customization of topics_pattern'
+        end
+      end
+
+      private
+      def check_for_schema_registry_connectivity_and_subjects
+        options = {}
+        if schema_registry_proxy && !schema_registry_proxy.empty?
+          options[:proxy] = schema_registry_proxy.to_s
+        end
+        if schema_registry_key and !schema_registry_key.empty?
+          options[:auth] = {:user => schema_registry_key, :password => schema_registry_secret.value}
+        end
+        client = Manticore::Client.new(options)
+
+        begin
+          response = client.get(@schema_registry_url.to_s + '/subjects').body
+        rescue Manticore::ManticoreException => e
+          raise LogStash::ConfigurationError.new("Schema registry service doesn't respond, error: #{e.message}")
+        end
+        registered_subjects = JSON.parse response
+        expected_subjects = @topics.map { |t| "#{t}-value"}
+        if (expected_subjects & registered_subjects).size != expected_subjects.size
+          undefined_topic_subjects = expected_subjects - registered_subjects
+          raise LogStash::ConfigurationError, "The schema registry does not contain definitions for required topic subjects: #{undefined_topic_subjects}"
+        end
+      end
+
+      def split_proxy_into_host_and_port(proxy_uri)
+        return nil unless proxy_uri && !proxy_uri.empty?
+
+        port = proxy_uri.port
+
+        host_spec = ""
+        host_spec << proxy_uri.scheme || "http"
+        host_spec << "://"
+        host_spec << "#{proxy_uri.userinfo}@" if proxy_uri.userinfo
+        host_spec << proxy_uri.host
+
+        [host_spec, port]
+      end
+
+      def check_for_key_and_secret
+        if schema_registry_key and !schema_registry_key.empty?
+          if !schema_registry_secret or schema_registry_secret.value.empty?
+            raise LogStash::ConfigurationError, "Setting `schema_registry_secret` is required when `schema_registry_key` is provided."
+          end
+        end
+      end
+    end
+  end
+end

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '10.5.3'
+  s.version         = '10.7.3'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+
@@ -46,6 +46,8 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-codec-json'
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'stud', '>= 0.0.22', '< 0.1.0'
+  s.add_runtime_dependency "manticore", '>= 0.5.4', '< 1.0.0'
+  s.add_runtime_dependency 'logstash-mixin-deprecation_logger_support', '~>1.0'
 
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'rspec-wait'

data/spec/integration/inputs/kafka_spec.rb

@@ -2,6 +2,9 @@
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/inputs/kafka"
 require "rspec/wait"
+require "stud/try"
+require "manticore"
+require "json"
 
 # Please run kafka_test_setup.sh prior to executing this integration test.
 describe "inputs/kafka", :integration => true do
@@ -33,7 +36,15 @@ describe "inputs/kafka", :integration => true do
   end
   let(:decorate_config) do
     { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_3,
-      'auto_offset_reset' => 'earliest', 'decorate_events' => true }
+      'auto_offset_reset' => 'earliest', 'decorate_events' => 'true' }
+  end
+  let(:decorate_headers_config) do
+    { 'topics' => ['logstash_integration_topic_plain_with_headers'], 'codec' => 'plain', 'group_id' => group_id_3,
+      'auto_offset_reset' => 'earliest', 'decorate_events' => 'extended' }
+  end
+  let(:decorate_bad_headers_config) do
+    { 'topics' => ['logstash_integration_topic_plain_with_headers_badly'], 'codec' => 'plain', 'group_id' => group_id_3,
+      'auto_offset_reset' => 'earliest', 'decorate_events' => 'extended' }
   end
   let(:manual_commit_config) do
     { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_5,
@@ -42,6 +53,35 @@ describe "inputs/kafka", :integration => true do
   let(:timeout_seconds) { 30 }
   let(:num_events) { 103 }
 
+  before(:all) do
+    # Prepare message with headers with valid UTF-8 chars
+    header = org.apache.kafka.common.header.internals.RecordHeader.new("name", "John ανδρεα €".to_java_bytes)
+    record = org.apache.kafka.clients.producer.ProducerRecord.new(
+                "logstash_integration_topic_plain_with_headers", 0, "key", "value", [header])
+    send_message(record)
+
+    # Prepare message with headers with invalid UTF-8 chars
+    invalid = "日本".encode('Shift_JIS').force_encoding(Encoding::UTF_8).to_java_bytes
+    header = org.apache.kafka.common.header.internals.RecordHeader.new("name", invalid)
+    record = org.apache.kafka.clients.producer.ProducerRecord.new(
+                "logstash_integration_topic_plain_with_headers_badly", 0, "key", "value", [header])
+
+    send_message(record)
+  end
+
+  def send_message(record)
+    props = java.util.Properties.new
+    kafka = org.apache.kafka.clients.producer.ProducerConfig
+    props.put(kafka::BOOTSTRAP_SERVERS_CONFIG, "localhost:9092")
+    props.put(kafka::KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer")
+    props.put(kafka::VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer")
+
+    producer = org.apache.kafka.clients.producer.KafkaProducer.new(props)
+
+    producer.send(record)
+    producer.close
+  end
+
   describe "#kafka-topics" do
 
     it "should consume all messages from plain 3-partition topic" do
@@ -71,7 +111,7 @@ describe "inputs/kafka", :integration => true do
 
   context "#kafka-topics-pattern" do
     it "should consume all messages from all 3 topics" do
-      total_events = num_events * 3
+      total_events = num_events * 3 + 2
       queue = consume_messages(pattern_config, timeout: timeout_seconds, event_count: total_events)
       expect(queue.length).to eq(total_events)
     end
@@ -88,6 +128,31 @@ describe "inputs/kafka", :integration => true do
         expect(event.get("[@metadata][kafka][timestamp]")).to be >= start
       end
     end
+
+    it "should show the right topic and group name in and kafka headers decorated kafka section" do
+      start = LogStash::Timestamp.now.time.to_i
+      consume_messages(decorate_headers_config, timeout: timeout_seconds, event_count: 1) do |queue, _|
+        expect(queue.length).to eq(1)
+        event = queue.shift
+        expect(event.get("[@metadata][kafka][topic]")).to eq("logstash_integration_topic_plain_with_headers")
+        expect(event.get("[@metadata][kafka][consumer_group]")).to eq(group_id_3)
+        expect(event.get("[@metadata][kafka][timestamp]")).to be >= start
+        expect(event.get("[@metadata][kafka][headers][name]")).to eq("John ανδρεα €")
+      end
+    end
+
+    it "should skip headers not encoded in UTF-8" do
+      start = LogStash::Timestamp.now.time.to_i
+      consume_messages(decorate_bad_headers_config, timeout: timeout_seconds, event_count: 1) do |queue, _|
+        expect(queue.length).to eq(1)
+        event = queue.shift
+        expect(event.get("[@metadata][kafka][topic]")).to eq("logstash_integration_topic_plain_with_headers_badly")
+        expect(event.get("[@metadata][kafka][consumer_group]")).to eq(group_id_3)
+        expect(event.get("[@metadata][kafka][timestamp]")).to be >= start
+
+        expect(event.include?("[@metadata][kafka][headers][name]")).to eq(false)
+      end
+    end
   end
 
   context "#kafka-offset-commit" do
@@ -120,20 +185,193 @@ describe "inputs/kafka", :integration => true do
       end
     end
   end
+end
+
+private
+
+def consume_messages(config, queue: Queue.new, timeout:, event_count:)
+  kafka_input = LogStash::Inputs::Kafka.new(config)
+  kafka_input.register
+  t = Thread.new { kafka_input.run(queue) }
+  begin
+    t.run
+    wait(timeout).for { queue.length }.to eq(event_count) unless timeout.eql?(false)
+    block_given? ? yield(queue, kafka_input) : queue
+  ensure
+    t.kill
+    t.join(30_000)
+  end
+end
 
-  private
 
-  def consume_messages(config, queue: Queue.new, timeout:, event_count:)
-    kafka_input = LogStash::Inputs::Kafka.new(config)
-    t = Thread.new { kafka_input.run(queue) }
-    begin
-      t.run
-      wait(timeout).for { queue.length }.to eq(event_count) unless timeout.eql?(false)
-      block_given? ? yield(queue, kafka_input) : queue
-    ensure
-      t.kill
-      t.join(30_000)
+describe "schema registry connection options" do
+  context "remote endpoint validation" do
+    it "should fail if not reachable" do
+      config = {'schema_registry_url' => 'http://localnothost:8081'}
+      kafka_input = LogStash::Inputs::Kafka.new(config)
+      expect { kafka_input.register }.to raise_error LogStash::ConfigurationError, /Schema registry service doesn't respond.*/
+    end
+
+    it "should fail if any topic is not matched by a subject on the schema registry" do
+      config = {
+        'schema_registry_url' => 'http://localhost:8081',
+        'topics' => ['temperature_stream']
+      }
+
+      kafka_input = LogStash::Inputs::Kafka.new(config)
+      expect { kafka_input.register }.to raise_error LogStash::ConfigurationError, /The schema registry does not contain definitions for required topic subjects: \["temperature_stream-value"\]/
+    end
+
+    context "register with subject present" do
+      SUBJECT_NAME = "temperature_stream-value"
+
+      before(:each) do
+        response = save_avro_schema_to_schema_registry(File.join(Dir.pwd, "spec", "unit", "inputs", "avro_schema_fixture_payment.asvc"), SUBJECT_NAME)
+        expect( response.code ).to be(200)
+      end
+
+      after(:each) do
+        schema_registry_client = Manticore::Client.new
+        delete_remote_schema(schema_registry_client, SUBJECT_NAME)
+      end
+
+      it "should correctly complete registration phase" do
+        config = {
+          'schema_registry_url' => 'http://localhost:8081',
+          'topics' => ['temperature_stream']
+        }
+        kafka_input = LogStash::Inputs::Kafka.new(config)
+        kafka_input.register
+      end
     end
   end
+end
 
+def save_avro_schema_to_schema_registry(schema_file, subject_name)
+  raw_schema = File.readlines(schema_file).map(&:chomp).join
+  raw_schema_quoted = raw_schema.gsub('"', '\"')
+  response = Manticore.post("http://localhost:8081/subjects/#{subject_name}/versions",
+          body: '{"schema": "' + raw_schema_quoted + '"}',
+          headers: {"Content-Type" => "application/vnd.schemaregistry.v1+json"})
+  response
 end
+
+def delete_remote_schema(schema_registry_client, subject_name)
+  expect(schema_registry_client.delete("http://localhost:8081/subjects/#{subject_name}").code ).to be(200)
+  expect(schema_registry_client.delete("http://localhost:8081/subjects/#{subject_name}?permanent=true").code ).to be(200)
+end
+
+# AdminClientConfig = org.alpache.kafka.clients.admin.AdminClientConfig
+
+describe "Schema registry API", :integration => true do
+
+  let(:schema_registry) { Manticore::Client.new }
+
+  context 'listing subject on clean instance' do
+    it "should return an empty set" do
+      subjects = JSON.parse schema_registry.get('http://localhost:8081/subjects').body
+      expect( subjects ).to be_empty
+    end
+  end
+
+  context 'send a schema definition' do
+    it "save the definition" do
+      response = save_avro_schema_to_schema_registry(File.join(Dir.pwd, "spec", "unit", "inputs", "avro_schema_fixture_payment.asvc"), "schema_test_1")
+      expect( response.code ).to be(200)
+      delete_remote_schema(schema_registry, "schema_test_1")
+    end
+
+    it "delete the schema just added" do
+      response = save_avro_schema_to_schema_registry(File.join(Dir.pwd, "spec", "unit", "inputs", "avro_schema_fixture_payment.asvc"), "schema_test_1")
+      expect( response.code ).to be(200)
+
+      expect( schema_registry.delete('http://localhost:8081/subjects/schema_test_1?permanent=false').code ).to be(200)
+      sleep(1)
+      subjects = JSON.parse schema_registry.get('http://localhost:8081/subjects').body
+      expect( subjects ).to be_empty
+    end
+  end
+
+  context 'use the schema to serialize' do
+    after(:each) do
+      expect( schema_registry.delete('http://localhost:8081/subjects/topic_avro-value').code ).to be(200)
+      sleep 1
+      expect( schema_registry.delete('http://localhost:8081/subjects/topic_avro-value?permanent=true').code ).to be(200)
+
+      Stud.try(3.times, [StandardError, RSpec::Expectations::ExpectationNotMetError]) do
+        wait(10).for do
+          subjects = JSON.parse schema_registry.get('http://localhost:8081/subjects').body
+          subjects.empty?
+        end.to be_truthy
+      end
+    end
+
+    let(:group_id_1) {rand(36**8).to_s(36)}
+
+    let(:avro_topic_name) { "topic_avro" }
+
+    let(:plain_config) do
+      { 'schema_registry_url' => 'http://localhost:8081',
+        'topics' => [avro_topic_name],
+        'codec' => 'plain',
+        'group_id' => group_id_1,
+        'auto_offset_reset' => 'earliest' }
+    end
+
+    def delete_topic_if_exists(topic_name)
+      props = java.util.Properties.new
+      props.put(Java::org.apache.kafka.clients.admin.AdminClientConfig::BOOTSTRAP_SERVERS_CONFIG, "localhost:9092")
+
+      admin_client = org.apache.kafka.clients.admin.AdminClient.create(props)
+      topics_list = admin_client.listTopics().names().get()
+      if topics_list.contains(topic_name)
+        result = admin_client.deleteTopics([topic_name])
+        result.values.get(topic_name).get()
+      end
+    end
+
+    def write_some_data_to(topic_name)
+      props = java.util.Properties.new
+      config = org.apache.kafka.clients.producer.ProducerConfig
+
+      serdes_config = Java::io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig
+      props.put(serdes_config::SCHEMA_REGISTRY_URL_CONFIG, "http://localhost:8081")
+
+      props.put(config::BOOTSTRAP_SERVERS_CONFIG, "localhost:9092")
+      props.put(config::KEY_SERIALIZER_CLASS_CONFIG, org.apache.kafka.common.serialization.StringSerializer.java_class)
+      props.put(config::VALUE_SERIALIZER_CLASS_CONFIG, Java::io.confluent.kafka.serializers.KafkaAvroSerializer.java_class)
+
+      parser = org.apache.avro.Schema::Parser.new()
+      user_schema = '''{"type":"record",
+                        "name":"myrecord",
+                        "fields":[
+                            {"name":"str_field", "type": "string"},
+                            {"name":"map_field", "type": {"type": "map", "values": "string"}}
+                        ]}'''
+      schema = parser.parse(user_schema)
+      avro_record = org.apache.avro.generic.GenericData::Record.new(schema)
+      avro_record.put("str_field", "value1")
+      avro_record.put("map_field", {"inner_field" => "inner value"})
+
+      producer = org.apache.kafka.clients.producer.KafkaProducer.new(props)
+      record = org.apache.kafka.clients.producer.ProducerRecord.new(topic_name, "avro_key", avro_record)
+      producer.send(record)
+    end
+
+    it "stored a new schema using Avro Kafka serdes" do
+      delete_topic_if_exists avro_topic_name
+      write_some_data_to avro_topic_name
+
+      subjects = JSON.parse schema_registry.get('http://localhost:8081/subjects').body
+      expect( subjects ).to contain_exactly("topic_avro-value")
+
+      num_events = 1
+      queue = consume_messages(plain_config, timeout: 30, event_count: num_events)
+      expect(queue.length).to eq(num_events)
+      elem = queue.pop
+      expect( elem.to_hash).not_to include("message")
+      expect( elem.get("str_field") ).to eq("value1")
+      expect( elem.get("map_field")["inner_field"] ).to eq("inner value")
+    end
+  end
+end

data/spec/unit/inputs/avro_schema_fixture_payment.asvc

@@ -0,0 +1,8 @@
+{"namespace": "io.confluent.examples.clients.basicavro",
+ "type": "record",
+ "name": "Payment",
+ "fields": [
+     {"name": "id", "type": "string"},
+     {"name": "amount", "type": "double"}
+ ]
+}

data/spec/unit/inputs/kafka_spec.rb

@@ -37,8 +37,41 @@ describe LogStash::Inputs::Kafka do
     expect { subject.register }.to_not raise_error
   end
 
+  context "register parameter verification" do
+    context "schema_registry_url" do
+      let(:config) do
+        { 'schema_registry_url' => 'http://localhost:8081', 'topics' => ['logstash'], 'consumer_threads' => 4 }
+      end
+
+      it "conflict with value_deserializer_class should fail" do
+        config['value_deserializer_class'] = 'my.fantasy.Deserializer'
+        expect { subject.register }.to raise_error LogStash::ConfigurationError, /Option schema_registry_url prohibit the customization of value_deserializer_class/
+      end
+
+      it "conflict with topics_pattern should fail" do
+        config['topics_pattern'] = 'topic_.*'
+        expect { subject.register }.to raise_error LogStash::ConfigurationError, /Option schema_registry_url prohibit the customization of topics_pattern/
+      end
+    end
+
+    context "decorate_events" do
+      let(:config) { { 'decorate_events' => 'extended'} }
+
+      it "should raise error for invalid value" do
+        config['decorate_events'] = 'avoid'
+        expect { subject.register }.to raise_error LogStash::ConfigurationError, /Something is wrong with your configuration./
+      end
+
+      it "should map old true boolean value to :record_props mode" do
+        config['decorate_events'] = "true"
+        subject.register
+        expect(subject.metadata_mode).to include(:record_props)
+      end
+    end
+  end
+
   context 'with client_rack' do
-    let(:config) { super.merge('client_rack' => 'EU-R1') }
+    let(:config) { super().merge('client_rack' => 'EU-R1') }
 
     it "sets broker rack parameter" do
       expect(org.apache.kafka.clients.consumer.KafkaConsumer).
@@ -50,7 +83,7 @@ describe LogStash::Inputs::Kafka do
   end
 
   context 'string integer config' do
-    let(:config) { super.merge('session_timeout_ms' => '25000', 'max_poll_interval_ms' => '345000') }
+    let(:config) { super().merge('session_timeout_ms' => '25000', 'max_poll_interval_ms' => '345000') }
 
     it "sets integer values" do
       expect(org.apache.kafka.clients.consumer.KafkaConsumer).
@@ -62,7 +95,7 @@ describe LogStash::Inputs::Kafka do
   end
 
   context 'integer config' do
-    let(:config) { super.merge('session_timeout_ms' => 25200, 'max_poll_interval_ms' => 123_000) }
+    let(:config) { super().merge('session_timeout_ms' => 25200, 'max_poll_interval_ms' => 123_000) }
 
     it "sets integer values" do
       expect(org.apache.kafka.clients.consumer.KafkaConsumer).
@@ -74,7 +107,7 @@ describe LogStash::Inputs::Kafka do
   end
 
   context 'string boolean config' do
-    let(:config) { super.merge('enable_auto_commit' => 'false', 'check_crcs' => 'true') }
+    let(:config) { super().merge('enable_auto_commit' => 'false', 'check_crcs' => 'true') }
 
     it "sets parameters" do
       expect(org.apache.kafka.clients.consumer.KafkaConsumer).
@@ -87,7 +120,7 @@ describe LogStash::Inputs::Kafka do
   end
 
   context 'boolean config' do
-    let(:config) { super.merge('enable_auto_commit' => true, 'check_crcs' => false) }
+    let(:config) { super().merge('enable_auto_commit' => true, 'check_crcs' => false) }
 
     it "sets parameters" do
       expect(org.apache.kafka.clients.consumer.KafkaConsumer).

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-kafka
 version: !ruby/object:Gem::Version
-  version: 10.5.3
+  version: 10.7.3
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-21 00:00:00.000000000 Z
+date: 2021-03-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -106,6 +106,40 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 0.1.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.5.4
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  name: manticore
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.5.4
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-deprecation_logger_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -183,17 +217,28 @@ files:
 - lib/logstash-integration-kafka_jars.rb
 - lib/logstash/inputs/kafka.rb
 - lib/logstash/outputs/kafka.rb
+- lib/logstash/plugin_mixins/common.rb
 - lib/logstash/plugin_mixins/kafka_support.rb
 - logstash-integration-kafka.gemspec
 - spec/fixtures/trust-store_stub.jks
 - spec/integration/inputs/kafka_spec.rb
 - spec/integration/outputs/kafka_spec.rb
+- spec/unit/inputs/avro_schema_fixture_payment.asvc
 - spec/unit/inputs/kafka_spec.rb
 - spec/unit/outputs/kafka_spec.rb
-- vendor/jar-dependencies/com/github/luben/zstd-jni/1.4.3-1/zstd-jni-1.4.3-1.jar
-- vendor/jar-dependencies/org/apache/kafka/kafka-clients/2.4.1/kafka-clients-2.4.1.jar
-- vendor/jar-dependencies/org/lz4/lz4-java/1.6.0/lz4-java-1.6.0.jar
-- vendor/jar-dependencies/org/slf4j/slf4j-api/1.7.28/slf4j-api-1.7.28.jar
+- vendor/jar-dependencies/com/github/luben/zstd-jni/1.4.4-7/zstd-jni-1.4.4-7.jar
+- vendor/jar-dependencies/io/confluent/common-config/5.5.1/common-config-5.5.1.jar
+- vendor/jar-dependencies/io/confluent/common-utils/5.5.1/common-utils-5.5.1.jar
+- vendor/jar-dependencies/io/confluent/kafka-avro-serializer/5.5.1/kafka-avro-serializer-5.5.1.jar
+- vendor/jar-dependencies/io/confluent/kafka-schema-registry-client/5.5.1/kafka-schema-registry-client-5.5.1.jar
+- vendor/jar-dependencies/io/confluent/kafka-schema-serializer/5.5.1/kafka-schema-serializer-5.5.1.jar
+- vendor/jar-dependencies/javax/ws/rs/javax.ws.rs-api/2.1.1/javax.ws.rs-api-2.1.1.jar
+- vendor/jar-dependencies/org/apache/avro/avro/1.9.2/avro-1.9.2.jar
+- vendor/jar-dependencies/org/apache/kafka/kafka-clients/2.5.1/kafka-clients-2.5.1.jar
+- vendor/jar-dependencies/org/apache/kafka/kafka_2.12/2.5.1/kafka_2.12-2.5.1.jar
+- vendor/jar-dependencies/org/glassfish/jersey/core/jersey-common/2.33/jersey-common-2.33.jar
+- vendor/jar-dependencies/org/lz4/lz4-java/1.7.1/lz4-java-1.7.1.jar
+- vendor/jar-dependencies/org/slf4j/slf4j-api/1.7.30/slf4j-api-1.7.30.jar
 - vendor/jar-dependencies/org/xerial/snappy/snappy-java/1.1.7.3/snappy-java-1.1.7.3.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
@@ -227,5 +272,6 @@ test_files:
 - spec/fixtures/trust-store_stub.jks
 - spec/integration/inputs/kafka_spec.rb
 - spec/integration/outputs/kafka_spec.rb
+- spec/unit/inputs/avro_schema_fixture_payment.asvc
 - spec/unit/inputs/kafka_spec.rb
 - spec/unit/outputs/kafka_spec.rb