logstash-integration-kafka 10.6.0-java → 10.7.4-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1d8b40d779e91e9c05dece65249660ab5c272b6833658cbca51b977d92936f42
-  data.tar.gz: 58323e216be645aede9f0b49c27824958b6627485bdf79a6774cd5f87b818245
+  metadata.gz: efc6c33cf871ecd41fc07468d3d6e47dc1a71c4dbd1800fe99127da703547dd2
+  data.tar.gz: 644f506705807c95e15fc035aac7f5d57233dd5809ae37f2c71939324ab9c3e7
 SHA512:
-  metadata.gz: 0f05eec028758745a2ab04b90d721128c088c46d0bb9a01923c389118c99a718a561d2ca2420fc2e206e4cf75e4e3695e1704a17d9eff9f4f18e66da3d3ccb85
-  data.tar.gz: 85e117a64d14d013674869ccadfb5487a04991ec83c6a9d6874496b862be200bf4e7203d3ac4fc779d67131ecbe82212fafd66884114337d17ed33dda7ad0963
+  metadata.gz: 28341e37050a860c8e87d0b74a4da4d1fdd37eff3e9c95bd5596dff4d9a613149f4dd09052a949f6002fb6da835601df12ca6d37d03ea25819ff6517833da37c
+  data.tar.gz: eb196288b02dd30b92bf3a4e083be0b0d8f005307203e20e8892262d46e48041e4d4212182be685ecae668970530c7050c7132f4c2aaa3be50f3f00e46a04122

data/CHANGELOG.md

@@ -1,3 +1,20 @@
+## 10.7.4
+  - Docs: make sure Kafka clients version is updated in docs [#83](https://github.com/logstash-plugins/logstash-integration-kafka/pull/83)
+    Since **10.6.0** Kafka client was updated to **2.5.1**
+
+## 10.7.3
+  - Changed `decorate_events` to add also Kafka headers [#78](https://github.com/logstash-plugins/logstash-integration-kafka/pull/78)
+
+## 10.7.2
+  - Update Jersey dependency to version 2.33 [#75](https://github.com/logstash-plugins/logstash-integration-kafka/pull/75)
+
+## 10.7.1
+  - Fix: dropped usage of SHUTDOWN event deprecated since Logstash 5.0 [#71](https://github.com/logstash-plugins/logstash-integration-kafka/pull/71)
+  
+## 10.7.0
+  - Switched use from Faraday to Manticore as HTTP client library to access Schema Registry service 
+    to fix issue [#63](https://github.com/logstash-plugins/logstash-integration-kafka/pull/63) 
+
 ## 10.6.0
   - Added functionality to Kafka input to use Avro deserializer in retrieving data from Kafka. The schema is retrieved
     from an instance of Confluent's Schema Registry service [#51](https://github.com/logstash-plugins/logstash-integration-kafka/pull/51)

data/README.md

@@ -1,6 +1,6 @@
 # Logstash Plugin
 
-[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-integration-kafka.svg)](https://travis-ci.org/logstash-plugins/logstash-integration-kafka)
+[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-integration-kafka.svg)](https://travis-ci.com/logstash-plugins/logstash-integration-kafka)
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 

data/docs/index.asciidoc

@@ -1,7 +1,7 @@
 :plugin: kafka
 :type: integration
 :no_codec:
-:kafka_client: 2.4
+:kafka_client: 2.5.1
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!

data/docs/input-kafka.asciidoc

@@ -2,8 +2,8 @@
 :plugin: kafka
 :type: input
 :default_codec: plain
-:kafka_client: 2.4
-:kafka_client_doc: 24
+:kafka_client: 2.5
+:kafka_client_doc: 25
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!
@@ -73,7 +73,7 @@ either when the record was created (default) or when it was received by the
 broker. See more about property log.message.timestamp.type at
 https://kafka.apache.org/{kafka_client_doc}/documentation.html#brokerconfigs
 
-Metadata is only added to the event if the `decorate_events` option is set to true (it defaults to false).
+Metadata is only added to the event if the `decorate_events` option is set to `basic` or `extended` (it defaults to `none`).
 
 Please note that `@metadata` fields are not part of any of your events at output time. If you need these information to be 
 inserted into your original event, you'll have to use the `mutate` filter to manually copy the required fields into your `event`.
@@ -99,7 +99,7 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai
 | <<plugins-{type}s-{plugin}-client_rack>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-connections_max_idle_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-consumer_threads>> |<<number,number>>|No
-| <<plugins-{type}s-{plugin}-decorate_events>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-decorate_events>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-enable_auto_commit>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-exclude_internal_topics>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-fetch_max_bytes>> |<<number,number>>|No
@@ -246,10 +246,16 @@ balance — more threads than partitions means that some threads will be idl
 [id="plugins-{type}s-{plugin}-decorate_events"]
 ===== `decorate_events` 
 
-  * Value type is <<boolean,boolean>>
-  * Default value is `false`
-
-Option to add Kafka metadata like topic, message size to the event.
+  * Value type is <<string,string>>
+  * Accepted values are:
+    - `none`: no metadata is added
+    - `basic`: record's attributes are added
+    - `extended`: record's attributes, headers are added
+    - `false`: deprecated alias for `none`
+    - `true`: deprecated alias for `basic`
+  * Default value is `none`
+
+Option to add Kafka metadata like topic, message size and header key values to the event.
 This will add a field named `kafka` to the logstash event containing the following attributes:
 
 * `topic`: The topic this message is associated with

data/docs/output-kafka.asciidoc

@@ -2,8 +2,8 @@
 :plugin: kafka
 :type: output
 :default_codec: plain
-:kafka_client: 2.4
-:kafka_client_doc: 24
+:kafka_client: 2.5
+:kafka_client_doc: 25
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!

data/lib/logstash-integration-kafka_jars.rb

@@ -9,7 +9,7 @@ require_jar('io.confluent', 'kafka-schema-registry-client', '5.5.1')
 require_jar('org.apache.kafka', 'kafka_2.12', '2.5.1')
 require_jar('io.confluent', 'common-utils', '5.5.1')
 require_jar('javax.ws.rs', 'javax.ws.rs-api', '2.1.1')
-require_jar('org.glassfish.jersey.core', 'jersey-common', '2.30')
+require_jar('org.glassfish.jersey.core', 'jersey-common', '2.33')
 require_jar('org.apache.kafka', 'kafka-clients', '2.5.1')
 require_jar('com.github.luben', 'zstd-jni', '1.4.4-7')
 require_jar('org.slf4j', 'slf4j-api', '1.7.30')

data/lib/logstash/inputs/kafka.rb

@@ -4,10 +4,11 @@ require 'stud/interval'
 require 'java'
 require 'logstash-integration-kafka_jars.rb'
 require 'logstash/plugin_mixins/kafka_support'
-require "faraday"
+require 'manticore'
 require "json"
 require "logstash/json"
 require_relative '../plugin_mixins/common'
+require 'logstash/plugin_mixins/deprecation_logger_support'
 
 # This input will read events from a Kafka topic. It uses the 0.10 version of
 # the consumer API provided by Kafka to read messages from the broker.
@@ -58,6 +59,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
 
   include LogStash::PluginMixins::KafkaSupport
   include ::LogStash::PluginMixins::KafkaAvroSchemaRegistry
+  include LogStash::PluginMixins::DeprecationLoggerSupport
 
   config_name 'kafka'
 
@@ -233,22 +235,49 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   config :sasl_jaas_config, :validate => :string
   # Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html
   config :kerberos_config, :validate => :path
-  # Option to add Kafka metadata like topic, message size to the event.
-  # This will add a field named `kafka` to the logstash event containing the following attributes:
+  # Option to add Kafka metadata like topic, message size and header key values to the event.
+  # With `basic` this will add a field named `kafka` to the logstash event containing the following attributes:
   #   `topic`: The topic this message is associated with
   #   `consumer_group`: The consumer group used to read in this event
   #   `partition`: The partition this message is associated with
   #   `offset`: The offset from the partition this message is associated with
   #   `key`: A ByteBuffer containing the message key
   #   `timestamp`: The timestamp of this message
-  config :decorate_events, :validate => :boolean, :default => false
+  # While with `extended` it adds also all the key values present in the Kafka header if the key is valid UTF-8 else
+  # silently skip it.
+  config :decorate_events, :validate => %w(none basic extended false true), :default => "none"
+
+  attr_reader :metadata_mode
 
   public
   def register
     @runner_threads = []
+    @metadata_mode = extract_metadata_level(@decorate_events)
     check_schema_registry_parameters
   end
 
+  METADATA_NONE     = Set[].freeze
+  METADATA_BASIC    = Set[:record_props].freeze
+  METADATA_EXTENDED = Set[:record_props, :headers].freeze
+  METADATA_DEPRECATION_MAP = { 'true' => 'basic', 'false' => 'none' }
+
+  private
+  def extract_metadata_level(decorate_events_setting)
+    metadata_enabled = decorate_events_setting
+
+    if METADATA_DEPRECATION_MAP.include?(metadata_enabled)
+      canonical_value = METADATA_DEPRECATION_MAP[metadata_enabled]
+      deprecation_logger.deprecated("Deprecated value `#{decorate_events_setting}` for `decorate_events` option; use `#{canonical_value}` instead.")
+      metadata_enabled = canonical_value
+    end
+
+    case metadata_enabled
+    when 'none'     then METADATA_NONE
+    when 'basic'    then METADATA_BASIC
+    when 'extended' then METADATA_EXTENDED
+    end
+  end
+
   public
   def run(logstash_queue)
     @runner_consumers = consumer_threads.times.map { |i| create_consumer("#{client_id}-#{i}") }
@@ -292,7 +321,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
                 end
                 event.remove("message")
               end
-              if @decorate_events
+              if @metadata_mode.include?(:record_props)
                 event.set("[@metadata][kafka][topic]", record.topic)
                 event.set("[@metadata][kafka][consumer_group]", @group_id)
                 event.set("[@metadata][kafka][partition]", record.partition)
@@ -300,6 +329,15 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
                 event.set("[@metadata][kafka][key]", record.key)
                 event.set("[@metadata][kafka][timestamp]", record.timestamp)
               end
+              if @metadata_mode.include?(:headers)
+                record.headers.each do |header|
+                  s = String.from_java_bytes(header.value)
+                  s.force_encoding(Encoding::UTF_8)
+                  if s.valid_encoding?
+                    event.set("[@metadata][kafka][headers]["+header.key+"]", s)
+                  end
+                end
+              end
               logstash_queue << event
             end
           end

data/lib/logstash/outputs/kafka.rb

@@ -224,7 +224,6 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
     end
 
     events.each do |event|
-      break if event == LogStash::SHUTDOWN
       @codec.encode(event)
     end
 

data/lib/logstash/plugin_mixins/common.rb

@@ -45,20 +45,21 @@ module LogStash
 
       private
       def check_for_schema_registry_connectivity_and_subjects
-        client = Faraday.new(@schema_registry_url.to_s) do |conn|
-          if schema_registry_proxy && !schema_registry_proxy.empty?
-            conn.proxy = schema_registry_proxy.to_s
-          end
-          if schema_registry_key and !schema_registry_key.empty?
-            conn.basic_auth(schema_registry_key, schema_registry_secret.value)
-          end
+        options = {}
+        if schema_registry_proxy && !schema_registry_proxy.empty?
+          options[:proxy] = schema_registry_proxy.to_s
         end
+        if schema_registry_key and !schema_registry_key.empty?
+          options[:auth] = {:user => schema_registry_key, :password => schema_registry_secret.value}
+        end
+        client = Manticore::Client.new(options)
+
         begin
-          response = client.get('/subjects')
-        rescue Faraday::Error => e
+          response = client.get(@schema_registry_url.to_s + '/subjects').body
+        rescue Manticore::ManticoreException => e
           raise LogStash::ConfigurationError.new("Schema registry service doesn't respond, error: #{e.message}")
         end
-        registered_subjects = JSON.parse response.body
+        registered_subjects = JSON.parse response
         expected_subjects = @topics.map { |t| "#{t}-value"}
         if (expected_subjects & registered_subjects).size != expected_subjects.size
           undefined_topic_subjects = expected_subjects - registered_subjects

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '10.6.0'
+  s.version         = '10.7.4'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+
@@ -46,6 +46,8 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-codec-json'
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'stud', '>= 0.0.22', '< 0.1.0'
+  s.add_runtime_dependency "manticore", '>= 0.5.4', '< 1.0.0'
+  s.add_runtime_dependency 'logstash-mixin-deprecation_logger_support', '~>1.0'
 
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'rspec-wait'

data/spec/integration/inputs/kafka_spec.rb

@@ -3,7 +3,7 @@ require "logstash/devutils/rspec/spec_helper"
 require "logstash/inputs/kafka"
 require "rspec/wait"
 require "stud/try"
-require "faraday"
+require "manticore"
 require "json"
 
 # Please run kafka_test_setup.sh prior to executing this integration test.
@@ -36,7 +36,15 @@ describe "inputs/kafka", :integration => true do
   end
   let(:decorate_config) do
     { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_3,
-      'auto_offset_reset' => 'earliest', 'decorate_events' => true }
+      'auto_offset_reset' => 'earliest', 'decorate_events' => 'true' }
+  end
+  let(:decorate_headers_config) do
+    { 'topics' => ['logstash_integration_topic_plain_with_headers'], 'codec' => 'plain', 'group_id' => group_id_3,
+      'auto_offset_reset' => 'earliest', 'decorate_events' => 'extended' }
+  end
+  let(:decorate_bad_headers_config) do
+    { 'topics' => ['logstash_integration_topic_plain_with_headers_badly'], 'codec' => 'plain', 'group_id' => group_id_3,
+      'auto_offset_reset' => 'earliest', 'decorate_events' => 'extended' }
   end
   let(:manual_commit_config) do
     { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_5,
@@ -45,6 +53,35 @@ describe "inputs/kafka", :integration => true do
   let(:timeout_seconds) { 30 }
   let(:num_events) { 103 }
 
+  before(:all) do
+    # Prepare message with headers with valid UTF-8 chars
+    header = org.apache.kafka.common.header.internals.RecordHeader.new("name", "John ανδρεα €".to_java_bytes)
+    record = org.apache.kafka.clients.producer.ProducerRecord.new(
+                "logstash_integration_topic_plain_with_headers", 0, "key", "value", [header])
+    send_message(record)
+
+    # Prepare message with headers with invalid UTF-8 chars
+    invalid = "日本".encode('Shift_JIS').force_encoding(Encoding::UTF_8).to_java_bytes
+    header = org.apache.kafka.common.header.internals.RecordHeader.new("name", invalid)
+    record = org.apache.kafka.clients.producer.ProducerRecord.new(
+                "logstash_integration_topic_plain_with_headers_badly", 0, "key", "value", [header])
+
+    send_message(record)
+  end
+
+  def send_message(record)
+    props = java.util.Properties.new
+    kafka = org.apache.kafka.clients.producer.ProducerConfig
+    props.put(kafka::BOOTSTRAP_SERVERS_CONFIG, "localhost:9092")
+    props.put(kafka::KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer")
+    props.put(kafka::VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer")
+
+    producer = org.apache.kafka.clients.producer.KafkaProducer.new(props)
+
+    producer.send(record)
+    producer.close
+  end
+
   describe "#kafka-topics" do
 
     it "should consume all messages from plain 3-partition topic" do
@@ -74,7 +111,7 @@ describe "inputs/kafka", :integration => true do
 
   context "#kafka-topics-pattern" do
     it "should consume all messages from all 3 topics" do
-      total_events = num_events * 3
+      total_events = num_events * 3 + 2
       queue = consume_messages(pattern_config, timeout: timeout_seconds, event_count: total_events)
       expect(queue.length).to eq(total_events)
     end
@@ -91,6 +128,31 @@ describe "inputs/kafka", :integration => true do
         expect(event.get("[@metadata][kafka][timestamp]")).to be >= start
       end
     end
+
+    it "should show the right topic and group name in and kafka headers decorated kafka section" do
+      start = LogStash::Timestamp.now.time.to_i
+      consume_messages(decorate_headers_config, timeout: timeout_seconds, event_count: 1) do |queue, _|
+        expect(queue.length).to eq(1)
+        event = queue.shift
+        expect(event.get("[@metadata][kafka][topic]")).to eq("logstash_integration_topic_plain_with_headers")
+        expect(event.get("[@metadata][kafka][consumer_group]")).to eq(group_id_3)
+        expect(event.get("[@metadata][kafka][timestamp]")).to be >= start
+        expect(event.get("[@metadata][kafka][headers][name]")).to eq("John ανδρεα €")
+      end
+    end
+
+    it "should skip headers not encoded in UTF-8" do
+      start = LogStash::Timestamp.now.time.to_i
+      consume_messages(decorate_bad_headers_config, timeout: timeout_seconds, event_count: 1) do |queue, _|
+        expect(queue.length).to eq(1)
+        event = queue.shift
+        expect(event.get("[@metadata][kafka][topic]")).to eq("logstash_integration_topic_plain_with_headers_badly")
+        expect(event.get("[@metadata][kafka][consumer_group]")).to eq(group_id_3)
+        expect(event.get("[@metadata][kafka][timestamp]")).to be >= start
+
+        expect(event.include?("[@metadata][kafka][headers][name]")).to eq(false)
+      end
+    end
   end
 
   context "#kafka-offset-commit" do
@@ -129,6 +191,7 @@ private
 
 def consume_messages(config, queue: Queue.new, timeout:, event_count:)
   kafka_input = LogStash::Inputs::Kafka.new(config)
+  kafka_input.register
   t = Thread.new { kafka_input.run(queue) }
   begin
     t.run
@@ -164,11 +227,11 @@ describe "schema registry connection options" do
 
       before(:each) do
         response = save_avro_schema_to_schema_registry(File.join(Dir.pwd, "spec", "unit", "inputs", "avro_schema_fixture_payment.asvc"), SUBJECT_NAME)
-        expect( response.status ).to be(200)
+        expect( response.code ).to be(200)
       end
 
       after(:each) do
-        schema_registry_client = Faraday.new('http://localhost:8081')
+        schema_registry_client = Manticore::Client.new
         delete_remote_schema(schema_registry_client, SUBJECT_NAME)
       end
 
@@ -187,26 +250,26 @@ end
 def save_avro_schema_to_schema_registry(schema_file, subject_name)
   raw_schema = File.readlines(schema_file).map(&:chomp).join
   raw_schema_quoted = raw_schema.gsub('"', '\"')
-  response = Faraday.post("http://localhost:8081/subjects/#{subject_name}/versions",
-          '{"schema": "' + raw_schema_quoted + '"}',
-          "Content-Type" => "application/vnd.schemaregistry.v1+json")
+  response = Manticore.post("http://localhost:8081/subjects/#{subject_name}/versions",
+          body: '{"schema": "' + raw_schema_quoted + '"}',
+          headers: {"Content-Type" => "application/vnd.schemaregistry.v1+json"})
   response
 end
 
 def delete_remote_schema(schema_registry_client, subject_name)
-  expect(schema_registry_client.delete("/subjects/#{subject_name}").status ).to be(200)
-  expect(schema_registry_client.delete("/subjects/#{subject_name}?permanent=true").status ).to be(200)
+  expect(schema_registry_client.delete("http://localhost:8081/subjects/#{subject_name}").code ).to be(200)
+  expect(schema_registry_client.delete("http://localhost:8081/subjects/#{subject_name}?permanent=true").code ).to be(200)
 end
 
 # AdminClientConfig = org.alpache.kafka.clients.admin.AdminClientConfig
 
 describe "Schema registry API", :integration => true do
 
-  let(:schema_registry) { Faraday.new('http://localhost:8081') }
+  let(:schema_registry) { Manticore::Client.new }
 
   context 'listing subject on clean instance' do
     it "should return an empty set" do
-      subjects = JSON.parse schema_registry.get('/subjects').body
+      subjects = JSON.parse schema_registry.get('http://localhost:8081/subjects').body
       expect( subjects ).to be_empty
     end
   end
@@ -214,30 +277,30 @@ describe "Schema registry API", :integration => true do
   context 'send a schema definition' do
     it "save the definition" do
       response = save_avro_schema_to_schema_registry(File.join(Dir.pwd, "spec", "unit", "inputs", "avro_schema_fixture_payment.asvc"), "schema_test_1")
-      expect( response.status ).to be(200)
+      expect( response.code ).to be(200)
       delete_remote_schema(schema_registry, "schema_test_1")
     end
 
     it "delete the schema just added" do
       response = save_avro_schema_to_schema_registry(File.join(Dir.pwd, "spec", "unit", "inputs", "avro_schema_fixture_payment.asvc"), "schema_test_1")
-      expect( response.status ).to be(200)
+      expect( response.code ).to be(200)
 
-      expect( schema_registry.delete('/subjects/schema_test_1?permanent=false').status ).to be(200)
+      expect( schema_registry.delete('http://localhost:8081/subjects/schema_test_1?permanent=false').code ).to be(200)
       sleep(1)
-      subjects = JSON.parse schema_registry.get('/subjects').body
+      subjects = JSON.parse schema_registry.get('http://localhost:8081/subjects').body
       expect( subjects ).to be_empty
     end
   end
 
   context 'use the schema to serialize' do
     after(:each) do
-      expect( schema_registry.delete('/subjects/topic_avro-value').status ).to be(200)
+      expect( schema_registry.delete('http://localhost:8081/subjects/topic_avro-value').code ).to be(200)
       sleep 1
-      expect( schema_registry.delete('/subjects/topic_avro-value?permanent=true').status ).to be(200)
+      expect( schema_registry.delete('http://localhost:8081/subjects/topic_avro-value?permanent=true').code ).to be(200)
 
       Stud.try(3.times, [StandardError, RSpec::Expectations::ExpectationNotMetError]) do
         wait(10).for do
-          subjects = JSON.parse schema_registry.get('/subjects').body
+          subjects = JSON.parse schema_registry.get('http://localhost:8081/subjects').body
           subjects.empty?
         end.to be_truthy
       end
@@ -299,7 +362,7 @@ describe "Schema registry API", :integration => true do
       delete_topic_if_exists avro_topic_name
       write_some_data_to avro_topic_name
 
-      subjects = JSON.parse schema_registry.get('/subjects').body
+      subjects = JSON.parse schema_registry.get('http://localhost:8081/subjects').body
       expect( subjects ).to contain_exactly("topic_avro-value")
 
       num_events = 1

data/spec/unit/inputs/kafka_spec.rb

@@ -38,23 +38,40 @@ describe LogStash::Inputs::Kafka do
   end
 
   context "register parameter verification" do
-    let(:config) do
-      { 'schema_registry_url' => 'http://localhost:8081', 'topics' => ['logstash'], 'consumer_threads' => 4 }
-    end
+    context "schema_registry_url" do
+      let(:config) do
+        { 'schema_registry_url' => 'http://localhost:8081', 'topics' => ['logstash'], 'consumer_threads' => 4 }
+      end
 
-    it "schema_registry_url conflict with value_deserializer_class should fail" do
-      config['value_deserializer_class'] = 'my.fantasy.Deserializer'
-      expect { subject.register }.to raise_error LogStash::ConfigurationError, /Option schema_registry_url prohibit the customization of value_deserializer_class/
+      it "conflict with value_deserializer_class should fail" do
+        config['value_deserializer_class'] = 'my.fantasy.Deserializer'
+        expect { subject.register }.to raise_error LogStash::ConfigurationError, /Option schema_registry_url prohibit the customization of value_deserializer_class/
+      end
+
+      it "conflict with topics_pattern should fail" do
+        config['topics_pattern'] = 'topic_.*'
+        expect { subject.register }.to raise_error LogStash::ConfigurationError, /Option schema_registry_url prohibit the customization of topics_pattern/
+      end
     end
 
-    it "schema_registry_url conflict with topics_pattern should fail" do
-      config['topics_pattern'] = 'topic_.*'
-      expect { subject.register }.to raise_error LogStash::ConfigurationError, /Option schema_registry_url prohibit the customization of topics_pattern/
+    context "decorate_events" do
+      let(:config) { { 'decorate_events' => 'extended'} }
+
+      it "should raise error for invalid value" do
+        config['decorate_events'] = 'avoid'
+        expect { subject.register }.to raise_error LogStash::ConfigurationError, /Something is wrong with your configuration./
+      end
+
+      it "should map old true boolean value to :record_props mode" do
+        config['decorate_events'] = "true"
+        subject.register
+        expect(subject.metadata_mode).to include(:record_props)
+      end
     end
   end
 
   context 'with client_rack' do
-    let(:config) { super.merge('client_rack' => 'EU-R1') }
+    let(:config) { super().merge('client_rack' => 'EU-R1') }
 
     it "sets broker rack parameter" do
       expect(org.apache.kafka.clients.consumer.KafkaConsumer).
@@ -66,7 +83,7 @@ describe LogStash::Inputs::Kafka do
   end
 
   context 'string integer config' do
-    let(:config) { super.merge('session_timeout_ms' => '25000', 'max_poll_interval_ms' => '345000') }
+    let(:config) { super().merge('session_timeout_ms' => '25000', 'max_poll_interval_ms' => '345000') }
 
     it "sets integer values" do
       expect(org.apache.kafka.clients.consumer.KafkaConsumer).
@@ -78,7 +95,7 @@ describe LogStash::Inputs::Kafka do
   end
 
   context 'integer config' do
-    let(:config) { super.merge('session_timeout_ms' => 25200, 'max_poll_interval_ms' => 123_000) }
+    let(:config) { super().merge('session_timeout_ms' => 25200, 'max_poll_interval_ms' => 123_000) }
 
     it "sets integer values" do
       expect(org.apache.kafka.clients.consumer.KafkaConsumer).
@@ -90,7 +107,7 @@ describe LogStash::Inputs::Kafka do
   end
 
   context 'string boolean config' do
-    let(:config) { super.merge('enable_auto_commit' => 'false', 'check_crcs' => 'true') }
+    let(:config) { super().merge('enable_auto_commit' => 'false', 'check_crcs' => 'true') }
 
     it "sets parameters" do
       expect(org.apache.kafka.clients.consumer.KafkaConsumer).
@@ -103,7 +120,7 @@ describe LogStash::Inputs::Kafka do
   end
 
   context 'boolean config' do
-    let(:config) { super.merge('enable_auto_commit' => true, 'check_crcs' => false) }
+    let(:config) { super().merge('enable_auto_commit' => true, 'check_crcs' => false) }
 
     it "sets parameters" do
       expect(org.apache.kafka.clients.consumer.KafkaConsumer).

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-kafka
 version: !ruby/object:Gem::Version
-  version: 10.6.0
+  version: 10.7.4
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-28 00:00:00.000000000 Z
+date: 2021-04-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -106,6 +106,40 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 0.1.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.5.4
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  name: manticore
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.5.4
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-deprecation_logger_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -202,7 +236,7 @@ files:
 - vendor/jar-dependencies/org/apache/avro/avro/1.9.2/avro-1.9.2.jar
 - vendor/jar-dependencies/org/apache/kafka/kafka-clients/2.5.1/kafka-clients-2.5.1.jar
 - vendor/jar-dependencies/org/apache/kafka/kafka_2.12/2.5.1/kafka_2.12-2.5.1.jar
-- vendor/jar-dependencies/org/glassfish/jersey/core/jersey-common/2.30/jersey-common-2.30.jar
+- vendor/jar-dependencies/org/glassfish/jersey/core/jersey-common/2.33/jersey-common-2.33.jar
 - vendor/jar-dependencies/org/lz4/lz4-java/1.7.1/lz4-java-1.7.1.jar
 - vendor/jar-dependencies/org/slf4j/slf4j-api/1.7.30/slf4j-api-1.7.30.jar
 - vendor/jar-dependencies/org/xerial/snappy/snappy-java/1.1.7.3/snappy-java-1.1.7.3.jar