logstash-integration-kafka 10.7.2-java → 10.7.7-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3ca014c7463762ed6c2f4be1a9c0dce13356980f7ddf244294dd02837862d54f
-  data.tar.gz: 1d68b07fa127636a6bba867ffa0fff6beefd5b1d0a7ba1419eedcea224a97fe5
+  metadata.gz: 7a9e872c7a49cd1eb9fc9709c10be72d1c9609b0f19ef5550a2dd7cb317a925e
+  data.tar.gz: b0541ab279e7b7fcea74f1a951e7916abf54adde8bae60e2c4b5897b6f4daadb
 SHA512:
-  metadata.gz: 97c32c1cdf91c205a61fefa6ec23544b45cbc66205418df85b2e7bb0c724a2a5672f658b947c086f38808b445d99bc053ee0651eae6b3da0e5cee6065f519c59
-  data.tar.gz: 6f67718001a257aafa339dde326192eef009a1bd74a1c1d4e8b37ca277362f95da3a56100819d9ec972a782e0631a5e4c91013d59efab720ea8e8c39b9d100a4
+  metadata.gz: 1e53d939b471032107afaeb2b7656bf6c755d803e1effbda487b953ae6cc0b79fd97458834d079168023e0ecfa8aeec67a474dc1a4a99336a7be10af3d1b6412
+  data.tar.gz: bea64e388923bab253278e46eccc0701c2bda8f3067cc1e429223fbc3f9cd3a797ced2e334755bbbe3bbf1cf711237359d85e1c6ec22cde822ced2285840fd9f

data/CHANGELOG.md

@@ -1,8 +1,26 @@
+## 10.7.7
+  - Fix: Correct the settings to allow basic auth to work properly, either by setting `schema_registry_key/secret` or embedding username/password in the
+    url [#94](https://github.com/logstash-plugins/logstash-integration-kafka/pull/94)
+
+## 10.7.6
+  - Test: specify development dependency version [#91](https://github.com/logstash-plugins/logstash-integration-kafka/pull/91)
+
+## 10.7.5
+  - Improved error handling in the input plugin to avoid errors 'escaping' from the plugin, and crashing the logstash
+    process [#87](https://github.com/logstash-plugins/logstash-integration-kafka/pull/87)
+
+## 10.7.4
+  - Docs: make sure Kafka clients version is updated in docs [#83](https://github.com/logstash-plugins/logstash-integration-kafka/pull/83)
+    Since **10.6.0** Kafka client was updated to **2.5.1**
+
+## 10.7.3
+  - Changed `decorate_events` to add also Kafka headers [#78](https://github.com/logstash-plugins/logstash-integration-kafka/pull/78)
+
 ## 10.7.2
   - Update Jersey dependency to version 2.33 [#75](https://github.com/logstash-plugins/logstash-integration-kafka/pull/75)
 
 ## 10.7.1
-  - Fix: dropped usage of SHUTDOWN event deprecated since Logstash 5.0 [#71](https://github.com/logstash-plugins/logstash-integration-kafka/issue/71)
+  - Fix: dropped usage of SHUTDOWN event deprecated since Logstash 5.0 [#71](https://github.com/logstash-plugins/logstash-integration-kafka/pull/71)
   
 ## 10.7.0
   - Switched use from Faraday to Manticore as HTTP client library to access Schema Registry service 

data/docs/index.asciidoc

@@ -1,7 +1,7 @@
 :plugin: kafka
 :type: integration
 :no_codec:
-:kafka_client: 2.4
+:kafka_client: 2.5.1
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!

data/docs/input-kafka.asciidoc

@@ -2,8 +2,8 @@
 :plugin: kafka
 :type: input
 :default_codec: plain
-:kafka_client: 2.4
-:kafka_client_doc: 24
+:kafka_client: 2.5
+:kafka_client_doc: 25
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!
@@ -73,7 +73,7 @@ either when the record was created (default) or when it was received by the
 broker. See more about property log.message.timestamp.type at
 https://kafka.apache.org/{kafka_client_doc}/documentation.html#brokerconfigs
 
-Metadata is only added to the event if the `decorate_events` option is set to true (it defaults to false).
+Metadata is only added to the event if the `decorate_events` option is set to `basic` or `extended` (it defaults to `none`).
 
 Please note that `@metadata` fields are not part of any of your events at output time. If you need these information to be 
 inserted into your original event, you'll have to use the `mutate` filter to manually copy the required fields into your `event`.
@@ -99,7 +99,7 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai
 | <<plugins-{type}s-{plugin}-client_rack>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-connections_max_idle_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-consumer_threads>> |<<number,number>>|No
-| <<plugins-{type}s-{plugin}-decorate_events>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-decorate_events>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-enable_auto_commit>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-exclude_internal_topics>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-fetch_max_bytes>> |<<number,number>>|No
@@ -246,10 +246,16 @@ balance — more threads than partitions means that some threads will be idl
 [id="plugins-{type}s-{plugin}-decorate_events"]
 ===== `decorate_events` 
 
-  * Value type is <<boolean,boolean>>
-  * Default value is `false`
-
-Option to add Kafka metadata like topic, message size to the event.
+  * Value type is <<string,string>>
+  * Accepted values are:
+    - `none`: no metadata is added
+    - `basic`: record's attributes are added
+    - `extended`: record's attributes, headers are added
+    - `false`: deprecated alias for `none`
+    - `true`: deprecated alias for `basic`
+  * Default value is `none`
+
+Option to add Kafka metadata like topic, message size and header key values to the event.
 This will add a field named `kafka` to the logstash event containing the following attributes:
 
 * `topic`: The topic this message is associated with

data/docs/output-kafka.asciidoc

@@ -2,8 +2,8 @@
 :plugin: kafka
 :type: output
 :default_codec: plain
-:kafka_client: 2.4
-:kafka_client_doc: 24
+:kafka_client: 2.5
+:kafka_client_doc: 25
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!

data/lib/logstash/inputs/kafka.rb

@@ -8,6 +8,7 @@ require 'manticore'
 require "json"
 require "logstash/json"
 require_relative '../plugin_mixins/common'
+require 'logstash/plugin_mixins/deprecation_logger_support'
 
 # This input will read events from a Kafka topic. It uses the 0.10 version of
 # the consumer API provided by Kafka to read messages from the broker.
@@ -58,6 +59,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
 
   include LogStash::PluginMixins::KafkaSupport
   include ::LogStash::PluginMixins::KafkaAvroSchemaRegistry
+  include LogStash::PluginMixins::DeprecationLoggerSupport
 
   config_name 'kafka'
 
@@ -233,27 +235,57 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   config :sasl_jaas_config, :validate => :string
   # Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html
   config :kerberos_config, :validate => :path
-  # Option to add Kafka metadata like topic, message size to the event.
-  # This will add a field named `kafka` to the logstash event containing the following attributes:
+  # Option to add Kafka metadata like topic, message size and header key values to the event.
+  # With `basic` this will add a field named `kafka` to the logstash event containing the following attributes:
   #   `topic`: The topic this message is associated with
   #   `consumer_group`: The consumer group used to read in this event
   #   `partition`: The partition this message is associated with
   #   `offset`: The offset from the partition this message is associated with
   #   `key`: A ByteBuffer containing the message key
   #   `timestamp`: The timestamp of this message
-  config :decorate_events, :validate => :boolean, :default => false
+  # While with `extended` it adds also all the key values present in the Kafka header if the key is valid UTF-8 else
+  # silently skip it.
+  config :decorate_events, :validate => %w(none basic extended false true), :default => "none"
+
+  attr_reader :metadata_mode
 
   public
   def register
     @runner_threads = []
+    @metadata_mode = extract_metadata_level(@decorate_events)
+    @pattern ||= java.util.regex.Pattern.compile(@topics_pattern) unless @topics_pattern.nil?
     check_schema_registry_parameters
   end
 
+  METADATA_NONE     = Set[].freeze
+  METADATA_BASIC    = Set[:record_props].freeze
+  METADATA_EXTENDED = Set[:record_props, :headers].freeze
+  METADATA_DEPRECATION_MAP = { 'true' => 'basic', 'false' => 'none' }
+
+  private
+  def extract_metadata_level(decorate_events_setting)
+    metadata_enabled = decorate_events_setting
+
+    if METADATA_DEPRECATION_MAP.include?(metadata_enabled)
+      canonical_value = METADATA_DEPRECATION_MAP[metadata_enabled]
+      deprecation_logger.deprecated("Deprecated value `#{decorate_events_setting}` for `decorate_events` option; use `#{canonical_value}` instead.")
+      metadata_enabled = canonical_value
+    end
+
+    case metadata_enabled
+    when 'none'     then METADATA_NONE
+    when 'basic'    then METADATA_BASIC
+    when 'extended' then METADATA_EXTENDED
+    end
+  end
+
   public
   def run(logstash_queue)
-    @runner_consumers = consumer_threads.times.map { |i| create_consumer("#{client_id}-#{i}") }
-    @runner_threads = @runner_consumers.map { |consumer| thread_runner(logstash_queue, consumer) }
-    @runner_threads.each { |t| t.join }
+    @runner_consumers = consumer_threads.times.map { |i| subscribe(create_consumer("#{client_id}-#{i}")) }
+    @runner_threads = @runner_consumers.map.with_index { |consumer, i| thread_runner(logstash_queue, consumer,
+                                                                                     "kafka-input-worker-#{client_id}-#{i}") }
+    @runner_threads.each(&:start)
+    @runner_threads.each(&:join)
   end # def run
 
   public
@@ -267,53 +299,100 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
     @runner_consumers
   end
 
-  private
-  def thread_runner(logstash_queue, consumer)
-    Thread.new do
+  def subscribe(consumer)
+    @pattern.nil? ? consumer.subscribe(topics) : consumer.subscribe(@pattern)
+    consumer
+  end
+
+  def thread_runner(logstash_queue, consumer, name)
+    java.lang.Thread.new do
+      LogStash::Util::set_thread_name(name)
       begin
-        unless @topics_pattern.nil?
-          nooplistener = org.apache.kafka.clients.consumer.internals.NoOpConsumerRebalanceListener.new
-          pattern = java.util.regex.Pattern.compile(@topics_pattern)
-          consumer.subscribe(pattern, nooplistener)
-        else
-          consumer.subscribe(topics);
-        end
         codec_instance = @codec.clone
-        while !stop?
-          records = consumer.poll(poll_timeout_ms)
-          next unless records.count > 0
-          for record in records do
-            codec_instance.decode(record.value.to_s) do |event|
-              decorate(event)
-              if schema_registry_url
-                json = LogStash::Json.load(record.value.to_s)
-                json.each do |k, v|
-                  event.set(k, v)
-                end
-                event.remove("message")
-              end
-              if @decorate_events
-                event.set("[@metadata][kafka][topic]", record.topic)
-                event.set("[@metadata][kafka][consumer_group]", @group_id)
-                event.set("[@metadata][kafka][partition]", record.partition)
-                event.set("[@metadata][kafka][offset]", record.offset)
-                event.set("[@metadata][kafka][key]", record.key)
-                event.set("[@metadata][kafka][timestamp]", record.timestamp)
-              end
-              logstash_queue << event
-            end
+        until stop?
+          records = do_poll(consumer)
+          unless records.empty?
+            records.each { |record| handle_record(record, codec_instance, logstash_queue) }
+            maybe_commit_offset(consumer)
           end
-          # Manual offset commit
-          consumer.commitSync if @enable_auto_commit.eql?(false)
         end
-      rescue org.apache.kafka.common.errors.WakeupException => e
-        raise e if !stop?
       ensure
         consumer.close
       end
     end
   end
 
+  def do_poll(consumer)
+    records = []
+    begin
+      records = consumer.poll(poll_timeout_ms)
+    rescue org.apache.kafka.common.errors.WakeupException => e
+      logger.debug("Wake up from poll", :kafka_error_message => e)
+      raise e unless stop?
+    rescue => e
+      logger.error("Unable to poll Kafka consumer",
+                   :kafka_error_message => e,
+                   :cause => e.respond_to?(:getCause) ? e.getCause : nil)
+      Stud.stoppable_sleep(1) { stop? }
+    end
+    records
+  end
+
+  def handle_record(record, codec_instance, queue)
+    codec_instance.decode(record.value.to_s) do |event|
+      decorate(event)
+      maybe_apply_schema(event, record)
+      maybe_set_metadata(event, record)
+      queue << event
+    end
+  end
+
+  def maybe_apply_schema(event, record)
+    if schema_registry_url
+      json = LogStash::Json.load(record.value.to_s)
+      json.each do |k, v|
+        event.set(k, v)
+      end
+      event.remove("message")
+    end
+  end
+
+  def maybe_set_metadata(event, record)
+    if @metadata_mode.include?(:record_props)
+      event.set("[@metadata][kafka][topic]", record.topic)
+      event.set("[@metadata][kafka][consumer_group]", @group_id)
+      event.set("[@metadata][kafka][partition]", record.partition)
+      event.set("[@metadata][kafka][offset]", record.offset)
+      event.set("[@metadata][kafka][key]", record.key)
+      event.set("[@metadata][kafka][timestamp]", record.timestamp)
+    end
+    if @metadata_mode.include?(:headers)
+      record.headers.each do |header|
+        s = String.from_java_bytes(header.value)
+        s.force_encoding(Encoding::UTF_8)
+        if s.valid_encoding?
+          event.set("[@metadata][kafka][headers][" + header.key + "]", s)
+        end
+      end
+    end
+  end
+
+  def maybe_commit_offset(consumer)
+    begin
+      consumer.commitSync if @enable_auto_commit.eql?(false)
+    rescue org.apache.kafka.common.errors.WakeupException => e
+      logger.debug("Wake up from commitSync", :kafka_error_message => e)
+      raise e unless stop?
+    rescue StandardError => e
+      # For transient errors, the commit should be successful after the next set of
+      # polled records has been processed.
+      # But, it might also be worth thinking about adding a configurable retry mechanism
+      logger.error("Unable to commit records",
+                   :kafka_error_message => e,
+                   :cause => e.respond_to?(:getCause) ? e.getCause() : nil)
+    end
+  end
+
   private
   def create_consumer(client_id)
     begin
@@ -354,13 +433,16 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
       if schema_registry_url
         props.put(kafka::VALUE_DESERIALIZER_CLASS_CONFIG, Java::io.confluent.kafka.serializers.KafkaAvroDeserializer.java_class)
         serdes_config = Java::io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig
-        props.put(serdes_config::SCHEMA_REGISTRY_URL_CONFIG, schema_registry_url.to_s)
+        props.put(serdes_config::SCHEMA_REGISTRY_URL_CONFIG, schema_registry_url.uri.to_s)
         if schema_registry_proxy && !schema_registry_proxy.empty?
           props.put(serdes_config::PROXY_HOST, @schema_registry_proxy_host)
           props.put(serdes_config::PROXY_PORT, @schema_registry_proxy_port)
         end
         if schema_registry_key && !schema_registry_key.empty?
+          props.put(serdes_config::BASIC_AUTH_CREDENTIALS_SOURCE, 'USER_INFO')
           props.put(serdes_config::USER_INFO_CONFIG, schema_registry_key + ":" + schema_registry_secret.value)
+        else
+          props.put(serdes_config::BASIC_AUTH_CREDENTIALS_SOURCE, 'URL')
         end
       end
       if security_protocol == "SSL"

data/lib/logstash/plugin_mixins/common.rb

@@ -53,9 +53,8 @@ module LogStash
           options[:auth] = {:user => schema_registry_key, :password => schema_registry_secret.value}
         end
         client = Manticore::Client.new(options)
-
         begin
-          response = client.get(@schema_registry_url.to_s + '/subjects').body
+          response = client.get(@schema_registry_url.uri.to_s + '/subjects').body
         rescue Manticore::ManticoreException => e
           raise LogStash::ConfigurationError.new("Schema registry service doesn't respond, error: #{e.message}")
         end

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '10.7.2'
+  s.version         = '10.7.7'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+
@@ -47,9 +47,11 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'stud', '>= 0.0.22', '< 0.1.0'
   s.add_runtime_dependency "manticore", '>= 0.5.4', '< 1.0.0'
+  s.add_runtime_dependency 'logstash-mixin-deprecation_logger_support', '~>1.0'
 
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'rspec-wait'
-  s.add_development_dependency 'ruby-kafka'
+  s.add_development_dependency 'digest-crc', '~> 0.5.1' # 0.6.0 started using a C-ext
+  s.add_development_dependency 'ruby-kafka' # depends on digest-crc
   s.add_development_dependency 'snappy'
 end

data/spec/check_docs_spec.rb

@@ -0,0 +1,36 @@
+# encoding: utf-8
+require 'logstash-integration-kafka_jars'
+
+describe "[DOCS]" do
+
+  let(:docs_files) do
+    ['index.asciidoc', 'input-kafka.asciidoc', 'output-kafka.asciidoc'].map { |name| File.join('docs', name) }
+  end
+
+  let(:kafka_version_properties) do
+    loader = java.lang.Thread.currentThread.getContextClassLoader
+    version = loader.getResource('kafka/kafka-version.properties')
+    fail "kafka-version.properties missing" unless version
+    properties = java.util.Properties.new
+    properties.load version.openStream
+    properties
+  end
+
+  it 'is sync-ed with Kafka client version' do
+    version = kafka_version_properties.get('version') # e.g. '2.5.1'
+
+    fails = docs_files.map do |file|
+      if line = File.readlines(file).find { |line| line.index(':kafka_client:') }
+        puts "found #{line.inspect} in #{file}" if $VERBOSE # e.g. ":kafka_client: 2.5\n"
+        if !version.start_with?(line.strip.split[1])
+          "documentation at #{file} is out of sync with kafka-clients version (#{version.inspect}), detected line: #{line.inspect}"
+        else
+          nil
+        end
+      end
+    end
+
+    fail "\n" + fails.join("\n") if fails.flatten.any?
+  end
+
+end

data/spec/fixtures/jaas.config

@@ -0,0 +1,5 @@
+SchemaRegistry-Props {
+  org.eclipse.jetty.jaas.spi.PropertyFileLoginModule required
+  file="build/confluent_platform/etc/schema-registry/pwd"
+  debug="true";
+};

data/spec/fixtures/pwd

@@ -0,0 +1,5 @@
+fred: OBF:1w8t1tvf1w261w8v1w1c1tvn1w8x,user,admin
+barney: changeme,user,developer
+admin:admin,admin
+betty: MD5:164c88b302622e17050af52c89945d44,user
+wilma: CRYPT:adpexzg3FUZAk,admin,sr-user

data/spec/integration/inputs/kafka_spec.rb

@@ -36,7 +36,15 @@ describe "inputs/kafka", :integration => true do
   end
   let(:decorate_config) do
     { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_3,
-      'auto_offset_reset' => 'earliest', 'decorate_events' => true }
+      'auto_offset_reset' => 'earliest', 'decorate_events' => 'true' }
+  end
+  let(:decorate_headers_config) do
+    { 'topics' => ['logstash_integration_topic_plain_with_headers'], 'codec' => 'plain', 'group_id' => group_id_3,
+      'auto_offset_reset' => 'earliest', 'decorate_events' => 'extended' }
+  end
+  let(:decorate_bad_headers_config) do
+    { 'topics' => ['logstash_integration_topic_plain_with_headers_badly'], 'codec' => 'plain', 'group_id' => group_id_3,
+      'auto_offset_reset' => 'earliest', 'decorate_events' => 'extended' }
   end
   let(:manual_commit_config) do
     { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_5,
@@ -45,6 +53,35 @@ describe "inputs/kafka", :integration => true do
   let(:timeout_seconds) { 30 }
   let(:num_events) { 103 }
 
+  before(:all) do
+    # Prepare message with headers with valid UTF-8 chars
+    header = org.apache.kafka.common.header.internals.RecordHeader.new("name", "John ανδρεα €".to_java_bytes)
+    record = org.apache.kafka.clients.producer.ProducerRecord.new(
+                "logstash_integration_topic_plain_with_headers", 0, "key", "value", [header])
+    send_message(record)
+
+    # Prepare message with headers with invalid UTF-8 chars
+    invalid = "日本".encode('Shift_JIS').force_encoding(Encoding::UTF_8).to_java_bytes
+    header = org.apache.kafka.common.header.internals.RecordHeader.new("name", invalid)
+    record = org.apache.kafka.clients.producer.ProducerRecord.new(
+                "logstash_integration_topic_plain_with_headers_badly", 0, "key", "value", [header])
+
+    send_message(record)
+  end
+
+  def send_message(record)
+    props = java.util.Properties.new
+    kafka = org.apache.kafka.clients.producer.ProducerConfig
+    props.put(kafka::BOOTSTRAP_SERVERS_CONFIG, "localhost:9092")
+    props.put(kafka::KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer")
+    props.put(kafka::VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer")
+
+    producer = org.apache.kafka.clients.producer.KafkaProducer.new(props)
+
+    producer.send(record)
+    producer.close
+  end
+
   describe "#kafka-topics" do
 
     it "should consume all messages from plain 3-partition topic" do
@@ -74,7 +111,7 @@ describe "inputs/kafka", :integration => true do
 
   context "#kafka-topics-pattern" do
     it "should consume all messages from all 3 topics" do
-      total_events = num_events * 3
+      total_events = num_events * 3 + 2
       queue = consume_messages(pattern_config, timeout: timeout_seconds, event_count: total_events)
       expect(queue.length).to eq(total_events)
     end
@@ -91,6 +128,31 @@ describe "inputs/kafka", :integration => true do
         expect(event.get("[@metadata][kafka][timestamp]")).to be >= start
       end
     end
+
+    it "should show the right topic and group name in and kafka headers decorated kafka section" do
+      start = LogStash::Timestamp.now.time.to_i
+      consume_messages(decorate_headers_config, timeout: timeout_seconds, event_count: 1) do |queue, _|
+        expect(queue.length).to eq(1)
+        event = queue.shift
+        expect(event.get("[@metadata][kafka][topic]")).to eq("logstash_integration_topic_plain_with_headers")
+        expect(event.get("[@metadata][kafka][consumer_group]")).to eq(group_id_3)
+        expect(event.get("[@metadata][kafka][timestamp]")).to be >= start
+        expect(event.get("[@metadata][kafka][headers][name]")).to eq("John ανδρεα €")
+      end
+    end
+
+    it "should skip headers not encoded in UTF-8" do
+      start = LogStash::Timestamp.now.time.to_i
+      consume_messages(decorate_bad_headers_config, timeout: timeout_seconds, event_count: 1) do |queue, _|
+        expect(queue.length).to eq(1)
+        event = queue.shift
+        expect(event.get("[@metadata][kafka][topic]")).to eq("logstash_integration_topic_plain_with_headers_badly")
+        expect(event.get("[@metadata][kafka][consumer_group]")).to eq(group_id_3)
+        expect(event.get("[@metadata][kafka][timestamp]")).to be >= start
+
+        expect(event.include?("[@metadata][kafka][headers][name]")).to eq(false)
+      end
+    end
   end
 
   context "#kafka-offset-commit" do
@@ -129,19 +191,31 @@ private
 
 def consume_messages(config, queue: Queue.new, timeout:, event_count:)
   kafka_input = LogStash::Inputs::Kafka.new(config)
+  kafka_input.register
   t = Thread.new { kafka_input.run(queue) }
   begin
     t.run
     wait(timeout).for { queue.length }.to eq(event_count) unless timeout.eql?(false)
     block_given? ? yield(queue, kafka_input) : queue
   ensure
+    kafka_input.do_stop
     t.kill
-    t.join(30_000)
+    t.join(30)
   end
 end
 
 
 describe "schema registry connection options" do
+  schema_registry = Manticore::Client.new
+  before (:all) do
+    shutdown_schema_registry
+    startup_schema_registry(schema_registry)
+  end
+
+  after(:all) do
+    shutdown_schema_registry
+  end
+
   context "remote endpoint validation" do
     it "should fail if not reachable" do
       config = {'schema_registry_url' => 'http://localnothost:8081'}
@@ -168,8 +242,7 @@ describe "schema registry connection options" do
       end
 
       after(:each) do
-        schema_registry_client = Manticore::Client.new
-        delete_remote_schema(schema_registry_client, SUBJECT_NAME)
+        delete_remote_schema(schema_registry, SUBJECT_NAME)
       end
 
       it "should correctly complete registration phase" do
@@ -200,9 +273,25 @@ end
 
 # AdminClientConfig = org.alpache.kafka.clients.admin.AdminClientConfig
 
+def startup_schema_registry(schema_registry, auth=false)
+  system('./stop_schema_registry.sh')
+  auth ? system('./start_auth_schema_registry.sh') : system('./start_schema_registry.sh')
+  url = auth ? "http://barney:changeme@localhost:8081" : "http://localhost:8081"
+  Stud.try(20.times, [Manticore::SocketException, StandardError, RSpec::Expectations::ExpectationNotMetError]) do
+    expect(schema_registry.get(url).code).to eq(200)
+  end
+end
+
 describe "Schema registry API", :integration => true do
+  schema_registry = Manticore::Client.new
+
+  before(:all) do
+    startup_schema_registry(schema_registry)
+  end
 
-  let(:schema_registry) { Manticore::Client.new }
+  after(:all) do
+    shutdown_schema_registry
+  end
 
   context 'listing subject on clean instance' do
     it "should return an empty set" do
@@ -228,37 +317,58 @@ describe "Schema registry API", :integration => true do
       expect( subjects ).to be_empty
     end
   end
+end
+
+def shutdown_schema_registry
+  system('./stop_schema_registry.sh')
+end
+
+describe "Deserializing with the schema registry", :integration => true do
+  schema_registry = Manticore::Client.new
+
+  shared_examples 'it reads from a topic using a schema registry' do |with_auth|
+
+    before(:all) do
+      shutdown_schema_registry
+      startup_schema_registry(schema_registry, with_auth)
+    end
+
+    after(:all) do
+      shutdown_schema_registry
+    end
 
-  context 'use the schema to serialize' do
     after(:each) do
-      expect( schema_registry.delete('http://localhost:8081/subjects/topic_avro-value').code ).to be(200)
+      expect( schema_registry.delete("#{subject_url}/#{avro_topic_name}-value").code ).to be(200)
       sleep 1
-      expect( schema_registry.delete('http://localhost:8081/subjects/topic_avro-value?permanent=true').code ).to be(200)
+      expect( schema_registry.delete("#{subject_url}/#{avro_topic_name}-value?permanent=true").code ).to be(200)
 
       Stud.try(3.times, [StandardError, RSpec::Expectations::ExpectationNotMetError]) do
         wait(10).for do
-          subjects = JSON.parse schema_registry.get('http://localhost:8081/subjects').body
+          subjects = JSON.parse schema_registry.get(subject_url).body
           subjects.empty?
         end.to be_truthy
       end
     end
 
-    let(:group_id_1) {rand(36**8).to_s(36)}
-
-    let(:avro_topic_name) { "topic_avro" }
-
-    let(:plain_config) do
-      { 'schema_registry_url' => 'http://localhost:8081',
-        'topics' => [avro_topic_name],
-        'codec' => 'plain',
-        'group_id' => group_id_1,
-        'auto_offset_reset' => 'earliest' }
+    let(:base_config) do
+      {
+          'topics' => [avro_topic_name],
+          'codec' => 'plain',
+          'group_id' => group_id_1,
+          'auto_offset_reset' => 'earliest'
+      }
     end
 
-    def delete_topic_if_exists(topic_name)
+    let(:group_id_1) {rand(36**8).to_s(36)}
+
+    def delete_topic_if_exists(topic_name, user = nil, password = nil)
       props = java.util.Properties.new
       props.put(Java::org.apache.kafka.clients.admin.AdminClientConfig::BOOTSTRAP_SERVERS_CONFIG, "localhost:9092")
-
+      serdes_config = Java::io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig
+      unless user.nil?
+        props.put(serdes_config::BASIC_AUTH_CREDENTIALS_SOURCE, 'USER_INFO')
+        props.put(serdes_config::USER_INFO_CONFIG,  "#{user}:#{password}")
+      end
       admin_client = org.apache.kafka.clients.admin.AdminClient.create(props)
       topics_list = admin_client.listTopics().names().get()
       if topics_list.contains(topic_name)
@@ -267,7 +377,7 @@ describe "Schema registry API", :integration => true do
       end
     end
 
-    def write_some_data_to(topic_name)
+    def write_some_data_to(topic_name, user = nil, password = nil)
       props = java.util.Properties.new
       config = org.apache.kafka.clients.producer.ProducerConfig
 
@@ -275,6 +385,10 @@ describe "Schema registry API", :integration => true do
       props.put(serdes_config::SCHEMA_REGISTRY_URL_CONFIG, "http://localhost:8081")
 
       props.put(config::BOOTSTRAP_SERVERS_CONFIG, "localhost:9092")
+      unless user.nil?
+        props.put(serdes_config::BASIC_AUTH_CREDENTIALS_SOURCE, 'USER_INFO')
+        props.put(serdes_config::USER_INFO_CONFIG,  "#{user}:#{password}")
+      end
       props.put(config::KEY_SERIALIZER_CLASS_CONFIG, org.apache.kafka.common.serialization.StringSerializer.java_class)
       props.put(config::VALUE_SERIALIZER_CLASS_CONFIG, Java::io.confluent.kafka.serializers.KafkaAvroSerializer.java_class)
 
@@ -296,11 +410,11 @@ describe "Schema registry API", :integration => true do
     end
 
     it "stored a new schema using Avro Kafka serdes" do
-      delete_topic_if_exists avro_topic_name
-      write_some_data_to avro_topic_name
+      auth ? delete_topic_if_exists(avro_topic_name, user, password) : delete_topic_if_exists(avro_topic_name)
+      auth ? write_some_data_to(avro_topic_name, user, password) : write_some_data_to(avro_topic_name)
 
-      subjects = JSON.parse schema_registry.get('http://localhost:8081/subjects').body
-      expect( subjects ).to contain_exactly("topic_avro-value")
+      subjects = JSON.parse schema_registry.get(subject_url).body
+      expect( subjects ).to contain_exactly("#{avro_topic_name}-value")
 
       num_events = 1
       queue = consume_messages(plain_config, timeout: 30, event_count: num_events)
@@ -311,4 +425,43 @@ describe "Schema registry API", :integration => true do
       expect( elem.get("map_field")["inner_field"] ).to eq("inner value")
     end
   end
+
+  context 'with an unauthed schema registry' do
+    let(:auth) { false }
+    let(:avro_topic_name) { "topic_avro" }
+    let(:subject_url) { "http://localhost:8081/subjects" }
+    let(:plain_config)  { base_config.merge!({'schema_registry_url' => "http://localhost:8081"}) }
+
+    it_behaves_like 'it reads from a topic using a schema registry', false
+  end
+
+  context 'with an authed schema registry' do
+    let(:auth) { true }
+    let(:user) { "barney" }
+    let(:password) { "changeme" }
+    let(:avro_topic_name) { "topic_avro_auth" }
+    let(:subject_url) { "http://#{user}:#{password}@localhost:8081/subjects" }
+
+    context 'using schema_registry_key' do
+      let(:plain_config) do
+        base_config.merge!({
+          'schema_registry_url' => "http://localhost:8081",
+          'schema_registry_key' => user,
+          'schema_registry_secret' => password
+        })
+      end
+
+      it_behaves_like 'it reads from a topic using a schema registry', true
+    end
+
+    context 'using schema_registry_url' do
+      let(:plain_config) do
+        base_config.merge!({
+          'schema_registry_url' => "http://#{user}:#{password}@localhost:8081"
+        })
+      end
+
+      it_behaves_like 'it reads from a topic using a schema registry', true
+    end
+  end
 end

data/spec/integration/outputs/kafka_spec.rb

@@ -44,7 +44,7 @@ describe "outputs/kafka", :integration => true do
   end
 
   context 'when outputting messages serialized as Byte Array' do
-    let(:test_topic) { 'topic1b' }
+    let(:test_topic) { 'logstash_integration_topicbytearray' }
     let(:num_events) { 3 }
 
     before :each do

data/spec/unit/inputs/kafka_spec.rb

@@ -3,53 +3,210 @@ require "logstash/devutils/rspec/spec_helper"
 require "logstash/inputs/kafka"
 require "concurrent"
 
-class MockConsumer
-  def initialize
-    @wake = Concurrent::AtomicBoolean.new(false)
-  end
 
-  def subscribe(topics)
-  end
-  
-  def poll(ms)
-    if @wake.value
-      raise org.apache.kafka.common.errors.WakeupException.new
-    else
-      10.times.map do
-        org.apache.kafka.clients.consumer.ConsumerRecord.new("logstash", 0, 0, "key", "value")
+describe LogStash::Inputs::Kafka do
+  let(:common_config) { { 'topics' => ['logstash'] } }
+  let(:config) { common_config }
+  let(:consumer_double) { double(:consumer) }
+  let(:needs_raise) { false }
+  let(:payload) {
+    10.times.map do
+      org.apache.kafka.clients.consumer.ConsumerRecord.new("logstash", 0, 0, "key", "value")
+    end
+  }
+  subject { LogStash::Inputs::Kafka.new(config) }
+
+  describe '#poll' do
+    before do
+      polled = false
+      allow(consumer_double).to receive(:poll) do
+        if polled
+          []
+        else
+          polled = true
+          payload
+        end
       end
     end
+
+    it 'should poll' do
+      expect(consumer_double).to receive(:poll)
+      expect(subject.do_poll(consumer_double)).to eq(payload)
+    end
+
+    it 'should return nil if Kafka Exception is encountered' do
+      expect(consumer_double).to receive(:poll).and_raise(org.apache.kafka.common.errors.TopicAuthorizationException.new(''))
+      expect(subject.do_poll(consumer_double)).to be_empty
+    end
+
+    it 'should not throw if Kafka Exception is encountered' do
+      expect(consumer_double).to receive(:poll).and_raise(org.apache.kafka.common.errors.TopicAuthorizationException.new(''))
+      expect{subject.do_poll(consumer_double)}.not_to raise_error
+    end
+
+    it 'should return no records if Assertion Error is encountered' do
+      expect(consumer_double).to receive(:poll).and_raise(java.lang.AssertionError.new(''))
+      expect{subject.do_poll(consumer_double)}.to raise_error(java.lang.AssertionError)
+    end
   end
 
-  def close
+  describe '#maybe_commit_offset' do
+    context 'with auto commit disabled' do
+      let(:config) { common_config.merge('enable_auto_commit' => false) }
+
+      it 'should call commit on the consumer' do
+        expect(consumer_double).to receive(:commitSync)
+        subject.maybe_commit_offset(consumer_double)
+      end
+      it 'should not throw if a Kafka Exception is encountered' do
+        expect(consumer_double).to receive(:commitSync).and_raise(org.apache.kafka.common.errors.TopicAuthorizationException.new(''))
+        expect{subject.maybe_commit_offset(consumer_double)}.not_to raise_error
+      end
+
+      it 'should throw if Assertion Error is encountered' do
+        expect(consumer_double).to receive(:commitSync).and_raise(java.lang.AssertionError.new(''))
+        expect{subject.maybe_commit_offset(consumer_double)}.to raise_error(java.lang.AssertionError)
+      end
+    end
+
+    context 'with auto commit enabled' do
+      let(:config) { common_config.merge('enable_auto_commit' => true) }
+
+      it 'should not call commit on the consumer' do
+        expect(consumer_double).not_to receive(:commitSync)
+        subject.maybe_commit_offset(consumer_double)
+      end
+    end
   end
 
-  def wakeup
-    @wake.make_true
+  describe '#register' do
+    it "should register" do
+      expect { subject.register }.to_not raise_error
+    end
   end
-end
 
-describe LogStash::Inputs::Kafka do
-  let(:config) { { 'topics' => ['logstash'], 'consumer_threads' => 4 } }
-  subject { LogStash::Inputs::Kafka.new(config) }
+  describe '#running' do
+    let(:q) { Queue.new }
+    let(:config) { common_config.merge('client_id' => 'test') }
+
+    before do
+      expect(subject).to receive(:create_consumer).once.and_return(consumer_double)
+      allow(consumer_double).to receive(:wakeup)
+      allow(consumer_double).to receive(:close)
+      allow(consumer_double).to receive(:subscribe)
+    end
 
-  it "should register" do
-    expect { subject.register }.to_not raise_error
+    context 'when running' do
+      before do
+        polled = false
+        allow(consumer_double).to receive(:poll) do
+          if polled
+            []
+          else
+            polled = true
+            payload
+          end
+        end
+
+        subject.register
+        t = Thread.new do
+          sleep(1)
+          subject.do_stop
+        end
+        subject.run(q)
+        t.join
+      end
+
+      it 'should process the correct number of events' do
+        expect(q.size).to eq(10)
+      end
+
+      it 'should set the consumer thread name' do
+        expect(subject.instance_variable_get('@runner_threads').first.get_name).to eq("kafka-input-worker-test-0")
+      end
+    end
+
+    context 'when errors are encountered during poll' do
+      before do
+        raised, polled = false
+        allow(consumer_double).to receive(:poll) do
+          unless raised
+            raised = true
+            raise exception
+          end
+          if polled
+            []
+          else
+            polled = true
+            payload
+          end
+        end
+
+        subject.register
+        t = Thread.new do
+          sleep 2
+          subject.do_stop
+        end
+        subject.run(q)
+        t.join
+      end
+
+      context "when a Kafka exception is raised" do
+        let(:exception) { org.apache.kafka.common.errors.TopicAuthorizationException.new('Invalid topic') }
+
+        it 'should poll successfully' do
+          expect(q.size).to eq(10)
+        end
+      end
+
+      context "when a StandardError is raised" do
+        let(:exception) { StandardError.new('Standard Error') }
+
+        it 'should retry and poll successfully' do
+          expect(q.size).to eq(10)
+        end
+      end
+
+      context "when a java error is raised" do
+        let(:exception) { java.lang.AssertionError.new('Fatal assertion') }
+
+        it "should not retry" do
+          expect(q.size).to eq(0)
+        end
+      end
+    end
   end
 
   context "register parameter verification" do
-    let(:config) do
-      { 'schema_registry_url' => 'http://localhost:8081', 'topics' => ['logstash'], 'consumer_threads' => 4 }
-    end
+    context "schema_registry_url" do
+      let(:config) do
+        { 'schema_registry_url' => 'http://localhost:8081', 'topics' => ['logstash'], 'consumer_threads' => 4 }
+      end
 
-    it "schema_registry_url conflict with value_deserializer_class should fail" do
-      config['value_deserializer_class'] = 'my.fantasy.Deserializer'
-      expect { subject.register }.to raise_error LogStash::ConfigurationError, /Option schema_registry_url prohibit the customization of value_deserializer_class/
+      it "conflict with value_deserializer_class should fail" do
+        config['value_deserializer_class'] = 'my.fantasy.Deserializer'
+        expect { subject.register }.to raise_error LogStash::ConfigurationError, /Option schema_registry_url prohibit the customization of value_deserializer_class/
+      end
+
+      it "conflict with topics_pattern should fail" do
+        config['topics_pattern'] = 'topic_.*'
+        expect { subject.register }.to raise_error LogStash::ConfigurationError, /Option schema_registry_url prohibit the customization of topics_pattern/
+      end
     end
 
-    it "schema_registry_url conflict with topics_pattern should fail" do
-      config['topics_pattern'] = 'topic_.*'
-      expect { subject.register }.to raise_error LogStash::ConfigurationError, /Option schema_registry_url prohibit the customization of topics_pattern/
+    context "decorate_events" do
+      let(:config) { { 'decorate_events' => 'extended'} }
+
+      it "should raise error for invalid value" do
+        config['decorate_events'] = 'avoid'
+        expect { subject.register }.to raise_error LogStash::ConfigurationError, /Something is wrong with your configuration./
+      end
+
+      it "should map old true boolean value to :record_props mode" do
+        config['decorate_events'] = "true"
+        subject.register
+        expect(subject.metadata_mode).to include(:record_props)
+      end
     end
   end
 

data/spec/unit/outputs/kafka_spec.rb

@@ -8,6 +8,8 @@ describe "outputs/kafka" do
   let (:event) { LogStash::Event.new({'message' => 'hello', 'topic_name' => 'my_topic', 'host' => '172.0.0.1',
                                       '@timestamp' => LogStash::Timestamp.now}) }
 
+  let(:future) { double('kafka producer future') }
+
   context 'when initializing' do
     it "should register" do
       output = LogStash::Plugin.lookup("output", "kafka").new(simple_kafka_config)
@@ -24,8 +26,8 @@ describe "outputs/kafka" do
 
   context 'when outputting messages' do
     it 'should send logstash event to kafka broker' do
-      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
-        .with(an_instance_of(org.apache.kafka.clients.producer.ProducerRecord)).and_call_original
+      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).
+          with(an_instance_of(org.apache.kafka.clients.producer.ProducerRecord))
       kafka = LogStash::Outputs::Kafka.new(simple_kafka_config)
       kafka.register
       kafka.multi_receive([event])
@@ -33,18 +35,18 @@ describe "outputs/kafka" do
 
     it 'should support Event#sprintf placeholders in topic_id' do
       topic_field = 'topic_name'
-      expect(org.apache.kafka.clients.producer.ProducerRecord).to receive(:new)
-        .with("my_topic", event.to_s).and_call_original
-      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).and_call_original
+      expect(org.apache.kafka.clients.producer.ProducerRecord).to receive(:new).
+          with("my_topic", event.to_s).and_call_original
+      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
       kafka = LogStash::Outputs::Kafka.new({'topic_id' => "%{#{topic_field}}"})
       kafka.register
       kafka.multi_receive([event])
     end
 
     it 'should support field referenced message_keys' do
-      expect(org.apache.kafka.clients.producer.ProducerRecord).to receive(:new)
-        .with("test", "172.0.0.1", event.to_s).and_call_original
-      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).and_call_original
+      expect(org.apache.kafka.clients.producer.ProducerRecord).to receive(:new).
+          with("test", "172.0.0.1", event.to_s).and_call_original
+      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
       kafka = LogStash::Outputs::Kafka.new(simple_kafka_config.merge({"message_key" => "%{host}"}))
       kafka.register
       kafka.multi_receive([event])
@@ -71,22 +73,24 @@ describe "outputs/kafka" do
     before do
       count = 0
       expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
-        .exactly(sendcount).times
-        .and_wrap_original do |m, *args|
+        .exactly(sendcount).times do
         if count < failcount # fail 'failcount' times in a row.
           count += 1
           # Pick an exception at random
           raise exception_classes.shuffle.first.new("injected exception for testing")
         else
-          m.call(*args) # call original
+          count = :done
+          future # return future
         end
       end
+      expect(future).to receive :get
     end
 
     it "should retry until successful" do
       kafka = LogStash::Outputs::Kafka.new(simple_kafka_config)
       kafka.register
       kafka.multi_receive([event])
+      sleep(1.0) # allow for future.get call
     end
   end
 
@@ -101,15 +105,13 @@ describe "outputs/kafka" do
 
     before do
       count = 0
-      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
-        .exactly(1).times
-        .and_wrap_original do |m, *args|
+      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).exactly(1).times do
         if count < failcount # fail 'failcount' times in a row.
           count += 1
           # Pick an exception at random
           raise exception_classes.shuffle.first.new("injected exception for testing")
         else
-          m.call(*args) # call original
+          fail 'unexpected producer#send invocation'
         end
       end
     end
@@ -131,25 +133,24 @@ describe "outputs/kafka" do
 
       it "should retry until successful" do
         count = 0
-
-        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
-              .exactly(sendcount).times
-              .and_wrap_original do |m, *args|
+        success = nil
+        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).exactly(sendcount).times do
           if count < failcount
             count += 1
             # inject some failures.
 
             # Return a custom Future that will raise an exception to simulate a Kafka send() problem.
             future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
-            future.run
-            future
           else
-            m.call(*args)
+            success = true
+            future = java.util.concurrent.FutureTask.new { nil } # return no-op future
           end
+          future.tap { Thread.start { future.run } }
         end
         kafka = LogStash::Outputs::Kafka.new(simple_kafka_config)
         kafka.register
         kafka.multi_receive([event])
+        expect( success ).to be true
       end
     end
 
@@ -158,9 +159,7 @@ describe "outputs/kafka" do
       let(:max_sends) { 1 }
 
       it "should should only send once" do
-        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
-                                                                                       .once
-                                                                                       .and_wrap_original do |m, *args|
+        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).once do
           # Always fail.
           future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
           future.run
@@ -172,9 +171,7 @@ describe "outputs/kafka" do
       end
 
       it 'should not sleep' do
-        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
-                                                                                       .once
-                                                                                       .and_wrap_original do |m, *args|
+        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).once do
           # Always fail.
           future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
           future.run
@@ -193,13 +190,10 @@ describe "outputs/kafka" do
       let(:max_sends) { retries + 1 }
 
       it "should give up after retries are exhausted" do
-        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
-              .at_most(max_sends).times
-              .and_wrap_original do |m, *args|
+        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).at_most(max_sends).times do
           # Always fail.
           future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
-          future.run
-          future
+          future.tap { Thread.start { future.run } }
         end
         kafka = LogStash::Outputs::Kafka.new(simple_kafka_config.merge("retries" => retries))
         kafka.register
@@ -207,9 +201,7 @@ describe "outputs/kafka" do
       end
 
       it 'should only sleep retries number of times' do
-        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
-                                                                                       .at_most(max_sends).times
-                                                                                       .and_wrap_original do |m, *args|
+        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).at_most(max_sends).times do
           # Always fail.
           future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
           future.run

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-kafka
 version: !ruby/object:Gem::Version
-  version: 10.7.2
+  version: 10.7.7
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-03-05 00:00:00.000000000 Z
+date: 2021-07-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -126,6 +126,20 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 1.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-deprecation_logger_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -154,6 +168,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.1
+  name: digest-crc
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.1
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -206,6 +234,9 @@ files:
 - lib/logstash/plugin_mixins/common.rb
 - lib/logstash/plugin_mixins/kafka_support.rb
 - logstash-integration-kafka.gemspec
+- spec/check_docs_spec.rb
+- spec/fixtures/jaas.config
+- spec/fixtures/pwd
 - spec/fixtures/trust-store_stub.jks
 - spec/integration/inputs/kafka_spec.rb
 - spec/integration/outputs/kafka_spec.rb
@@ -255,6 +286,9 @@ signing_key:
 specification_version: 4
 summary: Integration with Kafka - input and output plugins
 test_files:
+- spec/check_docs_spec.rb
+- spec/fixtures/jaas.config
+- spec/fixtures/pwd
 - spec/fixtures/trust-store_stub.jks
 - spec/integration/inputs/kafka_spec.rb
 - spec/integration/outputs/kafka_spec.rb