logstash-integration-kafka 10.7.1-java → 10.7.6-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d68bf1e5dada6ec2bea00b759962a1cf0db3b1555c38ffd2f5f4a84178a79d9e
-  data.tar.gz: d41de9544b85313e5f064d68304be2044a77837e3a55778836cb0f04da085b76
+  metadata.gz: fc9c66beb8b3a0074d0fa5090dc8896eedd796428e4b7644d8d96e159c7bed2f
+  data.tar.gz: 780f7aeac690ee7a23e954e94c2359d8e9527bcd1464dde007ce7e55c4a023f8
 SHA512:
-  metadata.gz: 7b25d38f7e6014889a03060e38f04bdfde157dcec9a7bb5567936cdae636c83448594cc53941980363b463521561ab27bc4d2bc1ad9c4d3f2a1f4f83cca11878
-  data.tar.gz: 88244076005fa307881cf3b5108fb0c2f274b8e03f9238fda2f629799ab4751c9d1c7ab09148d67894191045e676686d479c1612ce112d597654d576b5652b0f
+  metadata.gz: db2e1a6eefc076887de8b42944fe5e0d4ddf025915faf55a4f150710772b08c170e0b4bd629871ca89f4b45fa36b572eb15191c284dec5ad62409c228343e9c5
+  data.tar.gz: b1ae9060e3cf83f6b7c6ca8d949ab2f78696b7a983c7154d8db01e2004875ece63f202ac5778095112adf9597b2d54cedac0ed79682b59bb0bd1b9002265fd36

data/CHANGELOG.md

@@ -1,5 +1,22 @@
+## 10.7.6
+  - Test: specify development dependency version [#91](https://github.com/logstash-plugins/logstash-integration-kafka/pull/91)
+
+## 10.7.5
+  - Improved error handling in the input plugin to avoid errors 'escaping' from the plugin, and crashing the logstash
+    process [#87](https://github.com/logstash-plugins/logstash-integration-kafka/pull/87)
+
+## 10.7.4
+  - Docs: make sure Kafka clients version is updated in docs [#83](https://github.com/logstash-plugins/logstash-integration-kafka/pull/83)
+    Since **10.6.0** Kafka client was updated to **2.5.1**
+
+## 10.7.3
+  - Changed `decorate_events` to add also Kafka headers [#78](https://github.com/logstash-plugins/logstash-integration-kafka/pull/78)
+
+## 10.7.2
+  - Update Jersey dependency to version 2.33 [#75](https://github.com/logstash-plugins/logstash-integration-kafka/pull/75)
+
 ## 10.7.1
-  - Fix: dropped usage of SHUTDOWN event deprecated since Logstash 5.0 [#71](https://github.com/logstash-plugins/logstash-integration-kafka/issue/71)
+  - Fix: dropped usage of SHUTDOWN event deprecated since Logstash 5.0 [#71](https://github.com/logstash-plugins/logstash-integration-kafka/pull/71)
   
 ## 10.7.0
   - Switched use from Faraday to Manticore as HTTP client library to access Schema Registry service 

data/docs/index.asciidoc

@@ -1,7 +1,7 @@
 :plugin: kafka
 :type: integration
 :no_codec:
-:kafka_client: 2.4
+:kafka_client: 2.5.1
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!

data/docs/input-kafka.asciidoc

@@ -2,8 +2,8 @@
 :plugin: kafka
 :type: input
 :default_codec: plain
-:kafka_client: 2.4
-:kafka_client_doc: 24
+:kafka_client: 2.5
+:kafka_client_doc: 25
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!
@@ -73,7 +73,7 @@ either when the record was created (default) or when it was received by the
 broker. See more about property log.message.timestamp.type at
 https://kafka.apache.org/{kafka_client_doc}/documentation.html#brokerconfigs
 
-Metadata is only added to the event if the `decorate_events` option is set to true (it defaults to false).
+Metadata is only added to the event if the `decorate_events` option is set to `basic` or `extended` (it defaults to `none`).
 
 Please note that `@metadata` fields are not part of any of your events at output time. If you need these information to be 
 inserted into your original event, you'll have to use the `mutate` filter to manually copy the required fields into your `event`.
@@ -99,7 +99,7 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai
 | <<plugins-{type}s-{plugin}-client_rack>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-connections_max_idle_ms>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-consumer_threads>> |<<number,number>>|No
-| <<plugins-{type}s-{plugin}-decorate_events>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-decorate_events>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-enable_auto_commit>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-exclude_internal_topics>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-fetch_max_bytes>> |<<number,number>>|No
@@ -246,10 +246,16 @@ balance — more threads than partitions means that some threads will be idl
 [id="plugins-{type}s-{plugin}-decorate_events"]
 ===== `decorate_events` 
 
-  * Value type is <<boolean,boolean>>
-  * Default value is `false`
-
-Option to add Kafka metadata like topic, message size to the event.
+  * Value type is <<string,string>>
+  * Accepted values are:
+    - `none`: no metadata is added
+    - `basic`: record's attributes are added
+    - `extended`: record's attributes, headers are added
+    - `false`: deprecated alias for `none`
+    - `true`: deprecated alias for `basic`
+  * Default value is `none`
+
+Option to add Kafka metadata like topic, message size and header key values to the event.
 This will add a field named `kafka` to the logstash event containing the following attributes:
 
 * `topic`: The topic this message is associated with

data/docs/output-kafka.asciidoc

@@ -2,8 +2,8 @@
 :plugin: kafka
 :type: output
 :default_codec: plain
-:kafka_client: 2.4
-:kafka_client_doc: 24
+:kafka_client: 2.5
+:kafka_client_doc: 25
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!

data/lib/logstash-integration-kafka_jars.rb

@@ -9,7 +9,7 @@ require_jar('io.confluent', 'kafka-schema-registry-client', '5.5.1')
 require_jar('org.apache.kafka', 'kafka_2.12', '2.5.1')
 require_jar('io.confluent', 'common-utils', '5.5.1')
 require_jar('javax.ws.rs', 'javax.ws.rs-api', '2.1.1')
-require_jar('org.glassfish.jersey.core', 'jersey-common', '2.30')
+require_jar('org.glassfish.jersey.core', 'jersey-common', '2.33')
 require_jar('org.apache.kafka', 'kafka-clients', '2.5.1')
 require_jar('com.github.luben', 'zstd-jni', '1.4.4-7')
 require_jar('org.slf4j', 'slf4j-api', '1.7.30')

data/lib/logstash/inputs/kafka.rb

@@ -8,6 +8,7 @@ require 'manticore'
 require "json"
 require "logstash/json"
 require_relative '../plugin_mixins/common'
+require 'logstash/plugin_mixins/deprecation_logger_support'
 
 # This input will read events from a Kafka topic. It uses the 0.10 version of
 # the consumer API provided by Kafka to read messages from the broker.
@@ -58,6 +59,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
 
   include LogStash::PluginMixins::KafkaSupport
   include ::LogStash::PluginMixins::KafkaAvroSchemaRegistry
+  include LogStash::PluginMixins::DeprecationLoggerSupport
 
   config_name 'kafka'
 
@@ -233,27 +235,57 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   config :sasl_jaas_config, :validate => :string
   # Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html
   config :kerberos_config, :validate => :path
-  # Option to add Kafka metadata like topic, message size to the event.
-  # This will add a field named `kafka` to the logstash event containing the following attributes:
+  # Option to add Kafka metadata like topic, message size and header key values to the event.
+  # With `basic` this will add a field named `kafka` to the logstash event containing the following attributes:
   #   `topic`: The topic this message is associated with
   #   `consumer_group`: The consumer group used to read in this event
   #   `partition`: The partition this message is associated with
   #   `offset`: The offset from the partition this message is associated with
   #   `key`: A ByteBuffer containing the message key
   #   `timestamp`: The timestamp of this message
-  config :decorate_events, :validate => :boolean, :default => false
+  # While with `extended` it adds also all the key values present in the Kafka header if the key is valid UTF-8 else
+  # silently skip it.
+  config :decorate_events, :validate => %w(none basic extended false true), :default => "none"
+
+  attr_reader :metadata_mode
 
   public
   def register
     @runner_threads = []
+    @metadata_mode = extract_metadata_level(@decorate_events)
+    @pattern ||= java.util.regex.Pattern.compile(@topics_pattern) unless @topics_pattern.nil?
     check_schema_registry_parameters
   end
 
+  METADATA_NONE     = Set[].freeze
+  METADATA_BASIC    = Set[:record_props].freeze
+  METADATA_EXTENDED = Set[:record_props, :headers].freeze
+  METADATA_DEPRECATION_MAP = { 'true' => 'basic', 'false' => 'none' }
+
+  private
+  def extract_metadata_level(decorate_events_setting)
+    metadata_enabled = decorate_events_setting
+
+    if METADATA_DEPRECATION_MAP.include?(metadata_enabled)
+      canonical_value = METADATA_DEPRECATION_MAP[metadata_enabled]
+      deprecation_logger.deprecated("Deprecated value `#{decorate_events_setting}` for `decorate_events` option; use `#{canonical_value}` instead.")
+      metadata_enabled = canonical_value
+    end
+
+    case metadata_enabled
+    when 'none'     then METADATA_NONE
+    when 'basic'    then METADATA_BASIC
+    when 'extended' then METADATA_EXTENDED
+    end
+  end
+
   public
   def run(logstash_queue)
-    @runner_consumers = consumer_threads.times.map { |i| create_consumer("#{client_id}-#{i}") }
-    @runner_threads = @runner_consumers.map { |consumer| thread_runner(logstash_queue, consumer) }
-    @runner_threads.each { |t| t.join }
+    @runner_consumers = consumer_threads.times.map { |i| subscribe(create_consumer("#{client_id}-#{i}")) }
+    @runner_threads = @runner_consumers.map.with_index { |consumer, i| thread_runner(logstash_queue, consumer,
+                                                                                     "kafka-input-worker-#{client_id}-#{i}") }
+    @runner_threads.each(&:start)
+    @runner_threads.each(&:join)
   end # def run
 
   public
@@ -267,53 +299,100 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
     @runner_consumers
   end
 
-  private
-  def thread_runner(logstash_queue, consumer)
-    Thread.new do
+  def subscribe(consumer)
+    @pattern.nil? ? consumer.subscribe(topics) : consumer.subscribe(@pattern)
+    consumer
+  end
+
+  def thread_runner(logstash_queue, consumer, name)
+    java.lang.Thread.new do
+      LogStash::Util::set_thread_name(name)
       begin
-        unless @topics_pattern.nil?
-          nooplistener = org.apache.kafka.clients.consumer.internals.NoOpConsumerRebalanceListener.new
-          pattern = java.util.regex.Pattern.compile(@topics_pattern)
-          consumer.subscribe(pattern, nooplistener)
-        else
-          consumer.subscribe(topics);
-        end
         codec_instance = @codec.clone
-        while !stop?
-          records = consumer.poll(poll_timeout_ms)
-          next unless records.count > 0
-          for record in records do
-            codec_instance.decode(record.value.to_s) do |event|
-              decorate(event)
-              if schema_registry_url
-                json = LogStash::Json.load(record.value.to_s)
-                json.each do |k, v|
-                  event.set(k, v)
-                end
-                event.remove("message")
-              end
-              if @decorate_events
-                event.set("[@metadata][kafka][topic]", record.topic)
-                event.set("[@metadata][kafka][consumer_group]", @group_id)
-                event.set("[@metadata][kafka][partition]", record.partition)
-                event.set("[@metadata][kafka][offset]", record.offset)
-                event.set("[@metadata][kafka][key]", record.key)
-                event.set("[@metadata][kafka][timestamp]", record.timestamp)
-              end
-              logstash_queue << event
-            end
+        until stop?
+          records = do_poll(consumer)
+          unless records.empty?
+            records.each { |record| handle_record(record, codec_instance, logstash_queue) }
+            maybe_commit_offset(consumer)
           end
-          # Manual offset commit
-          consumer.commitSync if @enable_auto_commit.eql?(false)
         end
-      rescue org.apache.kafka.common.errors.WakeupException => e
-        raise e if !stop?
       ensure
         consumer.close
       end
     end
   end
 
+  def do_poll(consumer)
+    records = []
+    begin
+      records = consumer.poll(poll_timeout_ms)
+    rescue org.apache.kafka.common.errors.WakeupException => e
+      logger.debug("Wake up from poll", :kafka_error_message => e)
+      raise e unless stop?
+    rescue => e
+      logger.error("Unable to poll Kafka consumer",
+                   :kafka_error_message => e,
+                   :cause => e.respond_to?(:getCause) ? e.getCause : nil)
+      Stud.stoppable_sleep(1) { stop? }
+    end
+    records
+  end
+
+  def handle_record(record, codec_instance, queue)
+    codec_instance.decode(record.value.to_s) do |event|
+      decorate(event)
+      maybe_apply_schema(event, record)
+      maybe_set_metadata(event, record)
+      queue << event
+    end
+  end
+
+  def maybe_apply_schema(event, record)
+    if schema_registry_url
+      json = LogStash::Json.load(record.value.to_s)
+      json.each do |k, v|
+        event.set(k, v)
+      end
+      event.remove("message")
+    end
+  end
+
+  def maybe_set_metadata(event, record)
+    if @metadata_mode.include?(:record_props)
+      event.set("[@metadata][kafka][topic]", record.topic)
+      event.set("[@metadata][kafka][consumer_group]", @group_id)
+      event.set("[@metadata][kafka][partition]", record.partition)
+      event.set("[@metadata][kafka][offset]", record.offset)
+      event.set("[@metadata][kafka][key]", record.key)
+      event.set("[@metadata][kafka][timestamp]", record.timestamp)
+    end
+    if @metadata_mode.include?(:headers)
+      record.headers.each do |header|
+        s = String.from_java_bytes(header.value)
+        s.force_encoding(Encoding::UTF_8)
+        if s.valid_encoding?
+          event.set("[@metadata][kafka][headers][" + header.key + "]", s)
+        end
+      end
+    end
+  end
+
+  def maybe_commit_offset(consumer)
+    begin
+      consumer.commitSync if @enable_auto_commit.eql?(false)
+    rescue org.apache.kafka.common.errors.WakeupException => e
+      logger.debug("Wake up from commitSync", :kafka_error_message => e)
+      raise e unless stop?
+    rescue StandardError => e
+      # For transient errors, the commit should be successful after the next set of
+      # polled records has been processed.
+      # But, it might also be worth thinking about adding a configurable retry mechanism
+      logger.error("Unable to commit records",
+                   :kafka_error_message => e,
+                   :cause => e.respond_to?(:getCause) ? e.getCause() : nil)
+    end
+  end
+
   private
   def create_consumer(client_id)
     begin

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '10.7.1'
+  s.version         = '10.7.6'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+
@@ -47,9 +47,11 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'stud', '>= 0.0.22', '< 0.1.0'
   s.add_runtime_dependency "manticore", '>= 0.5.4', '< 1.0.0'
+  s.add_runtime_dependency 'logstash-mixin-deprecation_logger_support', '~>1.0'
 
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'rspec-wait'
-  s.add_development_dependency 'ruby-kafka'
+  s.add_development_dependency 'digest-crc', '~> 0.5.1' # 0.6.0 started using a C-ext
+  s.add_development_dependency 'ruby-kafka' # depends on digest-crc
   s.add_development_dependency 'snappy'
 end

data/spec/check_docs_spec.rb

@@ -0,0 +1,36 @@
+# encoding: utf-8
+require 'logstash-integration-kafka_jars'
+
+describe "[DOCS]" do
+
+  let(:docs_files) do
+    ['index.asciidoc', 'input-kafka.asciidoc', 'output-kafka.asciidoc'].map { |name| File.join('docs', name) }
+  end
+
+  let(:kafka_version_properties) do
+    loader = java.lang.Thread.currentThread.getContextClassLoader
+    version = loader.getResource('kafka/kafka-version.properties')
+    fail "kafka-version.properties missing" unless version
+    properties = java.util.Properties.new
+    properties.load version.openStream
+    properties
+  end
+
+  it 'is sync-ed with Kafka client version' do
+    version = kafka_version_properties.get('version') # e.g. '2.5.1'
+
+    fails = docs_files.map do |file|
+      if line = File.readlines(file).find { |line| line.index(':kafka_client:') }
+        puts "found #{line.inspect} in #{file}" if $VERBOSE # e.g. ":kafka_client: 2.5\n"
+        if !version.start_with?(line.strip.split[1])
+          "documentation at #{file} is out of sync with kafka-clients version (#{version.inspect}), detected line: #{line.inspect}"
+        else
+          nil
+        end
+      end
+    end
+
+    fail "\n" + fails.join("\n") if fails.flatten.any?
+  end
+
+end

data/spec/integration/inputs/kafka_spec.rb

@@ -36,7 +36,15 @@ describe "inputs/kafka", :integration => true do
   end
   let(:decorate_config) do
     { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_3,
-      'auto_offset_reset' => 'earliest', 'decorate_events' => true }
+      'auto_offset_reset' => 'earliest', 'decorate_events' => 'true' }
+  end
+  let(:decorate_headers_config) do
+    { 'topics' => ['logstash_integration_topic_plain_with_headers'], 'codec' => 'plain', 'group_id' => group_id_3,
+      'auto_offset_reset' => 'earliest', 'decorate_events' => 'extended' }
+  end
+  let(:decorate_bad_headers_config) do
+    { 'topics' => ['logstash_integration_topic_plain_with_headers_badly'], 'codec' => 'plain', 'group_id' => group_id_3,
+      'auto_offset_reset' => 'earliest', 'decorate_events' => 'extended' }
   end
   let(:manual_commit_config) do
     { 'topics' => ['logstash_integration_topic_plain'], 'codec' => 'plain', 'group_id' => group_id_5,
@@ -45,6 +53,35 @@ describe "inputs/kafka", :integration => true do
   let(:timeout_seconds) { 30 }
   let(:num_events) { 103 }
 
+  before(:all) do
+    # Prepare message with headers with valid UTF-8 chars
+    header = org.apache.kafka.common.header.internals.RecordHeader.new("name", "John ανδρεα €".to_java_bytes)
+    record = org.apache.kafka.clients.producer.ProducerRecord.new(
+                "logstash_integration_topic_plain_with_headers", 0, "key", "value", [header])
+    send_message(record)
+
+    # Prepare message with headers with invalid UTF-8 chars
+    invalid = "日本".encode('Shift_JIS').force_encoding(Encoding::UTF_8).to_java_bytes
+    header = org.apache.kafka.common.header.internals.RecordHeader.new("name", invalid)
+    record = org.apache.kafka.clients.producer.ProducerRecord.new(
+                "logstash_integration_topic_plain_with_headers_badly", 0, "key", "value", [header])
+
+    send_message(record)
+  end
+
+  def send_message(record)
+    props = java.util.Properties.new
+    kafka = org.apache.kafka.clients.producer.ProducerConfig
+    props.put(kafka::BOOTSTRAP_SERVERS_CONFIG, "localhost:9092")
+    props.put(kafka::KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer")
+    props.put(kafka::VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer")
+
+    producer = org.apache.kafka.clients.producer.KafkaProducer.new(props)
+
+    producer.send(record)
+    producer.close
+  end
+
   describe "#kafka-topics" do
 
     it "should consume all messages from plain 3-partition topic" do
@@ -74,7 +111,7 @@ describe "inputs/kafka", :integration => true do
 
   context "#kafka-topics-pattern" do
     it "should consume all messages from all 3 topics" do
-      total_events = num_events * 3
+      total_events = num_events * 3 + 2
       queue = consume_messages(pattern_config, timeout: timeout_seconds, event_count: total_events)
       expect(queue.length).to eq(total_events)
     end
@@ -91,6 +128,31 @@ describe "inputs/kafka", :integration => true do
         expect(event.get("[@metadata][kafka][timestamp]")).to be >= start
       end
     end
+
+    it "should show the right topic and group name in and kafka headers decorated kafka section" do
+      start = LogStash::Timestamp.now.time.to_i
+      consume_messages(decorate_headers_config, timeout: timeout_seconds, event_count: 1) do |queue, _|
+        expect(queue.length).to eq(1)
+        event = queue.shift
+        expect(event.get("[@metadata][kafka][topic]")).to eq("logstash_integration_topic_plain_with_headers")
+        expect(event.get("[@metadata][kafka][consumer_group]")).to eq(group_id_3)
+        expect(event.get("[@metadata][kafka][timestamp]")).to be >= start
+        expect(event.get("[@metadata][kafka][headers][name]")).to eq("John ανδρεα €")
+      end
+    end
+
+    it "should skip headers not encoded in UTF-8" do
+      start = LogStash::Timestamp.now.time.to_i
+      consume_messages(decorate_bad_headers_config, timeout: timeout_seconds, event_count: 1) do |queue, _|
+        expect(queue.length).to eq(1)
+        event = queue.shift
+        expect(event.get("[@metadata][kafka][topic]")).to eq("logstash_integration_topic_plain_with_headers_badly")
+        expect(event.get("[@metadata][kafka][consumer_group]")).to eq(group_id_3)
+        expect(event.get("[@metadata][kafka][timestamp]")).to be >= start
+
+        expect(event.include?("[@metadata][kafka][headers][name]")).to eq(false)
+      end
+    end
   end
 
   context "#kafka-offset-commit" do
@@ -129,14 +191,16 @@ private
 
 def consume_messages(config, queue: Queue.new, timeout:, event_count:)
   kafka_input = LogStash::Inputs::Kafka.new(config)
+  kafka_input.register
   t = Thread.new { kafka_input.run(queue) }
   begin
     t.run
     wait(timeout).for { queue.length }.to eq(event_count) unless timeout.eql?(false)
     block_given? ? yield(queue, kafka_input) : queue
   ensure
+    kafka_input.do_stop
     t.kill
-    t.join(30_000)
+    t.join(30)
   end
 end
 

data/spec/integration/outputs/kafka_spec.rb

@@ -44,7 +44,7 @@ describe "outputs/kafka", :integration => true do
   end
 
   context 'when outputting messages serialized as Byte Array' do
-    let(:test_topic) { 'topic1b' }
+    let(:test_topic) { 'logstash_integration_topicbytearray' }
     let(:num_events) { 3 }
 
     before :each do

data/spec/unit/inputs/kafka_spec.rb

@@ -3,58 +3,215 @@ require "logstash/devutils/rspec/spec_helper"
 require "logstash/inputs/kafka"
 require "concurrent"
 
-class MockConsumer
-  def initialize
-    @wake = Concurrent::AtomicBoolean.new(false)
-  end
 
-  def subscribe(topics)
-  end
-  
-  def poll(ms)
-    if @wake.value
-      raise org.apache.kafka.common.errors.WakeupException.new
-    else
-      10.times.map do
-        org.apache.kafka.clients.consumer.ConsumerRecord.new("logstash", 0, 0, "key", "value")
+describe LogStash::Inputs::Kafka do
+  let(:common_config) { { 'topics' => ['logstash'] } }
+  let(:config) { common_config }
+  let(:consumer_double) { double(:consumer) }
+  let(:needs_raise) { false }
+  let(:payload) {
+    10.times.map do
+      org.apache.kafka.clients.consumer.ConsumerRecord.new("logstash", 0, 0, "key", "value")
+    end
+  }
+  subject { LogStash::Inputs::Kafka.new(config) }
+
+  describe '#poll' do
+    before do
+      polled = false
+      allow(consumer_double).to receive(:poll) do
+        if polled
+          []
+        else
+          polled = true
+          payload
+        end
       end
     end
+
+    it 'should poll' do
+      expect(consumer_double).to receive(:poll)
+      expect(subject.do_poll(consumer_double)).to eq(payload)
+    end
+
+    it 'should return nil if Kafka Exception is encountered' do
+      expect(consumer_double).to receive(:poll).and_raise(org.apache.kafka.common.errors.TopicAuthorizationException.new(''))
+      expect(subject.do_poll(consumer_double)).to be_empty
+    end
+
+    it 'should not throw if Kafka Exception is encountered' do
+      expect(consumer_double).to receive(:poll).and_raise(org.apache.kafka.common.errors.TopicAuthorizationException.new(''))
+      expect{subject.do_poll(consumer_double)}.not_to raise_error
+    end
+
+    it 'should return no records if Assertion Error is encountered' do
+      expect(consumer_double).to receive(:poll).and_raise(java.lang.AssertionError.new(''))
+      expect{subject.do_poll(consumer_double)}.to raise_error(java.lang.AssertionError)
+    end
   end
 
-  def close
+  describe '#maybe_commit_offset' do
+    context 'with auto commit disabled' do
+      let(:config) { common_config.merge('enable_auto_commit' => false) }
+
+      it 'should call commit on the consumer' do
+        expect(consumer_double).to receive(:commitSync)
+        subject.maybe_commit_offset(consumer_double)
+      end
+      it 'should not throw if a Kafka Exception is encountered' do
+        expect(consumer_double).to receive(:commitSync).and_raise(org.apache.kafka.common.errors.TopicAuthorizationException.new(''))
+        expect{subject.maybe_commit_offset(consumer_double)}.not_to raise_error
+      end
+
+      it 'should throw if Assertion Error is encountered' do
+        expect(consumer_double).to receive(:commitSync).and_raise(java.lang.AssertionError.new(''))
+        expect{subject.maybe_commit_offset(consumer_double)}.to raise_error(java.lang.AssertionError)
+      end
+    end
+
+    context 'with auto commit enabled' do
+      let(:config) { common_config.merge('enable_auto_commit' => true) }
+
+      it 'should not call commit on the consumer' do
+        expect(consumer_double).not_to receive(:commitSync)
+        subject.maybe_commit_offset(consumer_double)
+      end
+    end
   end
 
-  def wakeup
-    @wake.make_true
+  describe '#register' do
+    it "should register" do
+      expect { subject.register }.to_not raise_error
+    end
   end
-end
 
-describe LogStash::Inputs::Kafka do
-  let(:config) { { 'topics' => ['logstash'], 'consumer_threads' => 4 } }
-  subject { LogStash::Inputs::Kafka.new(config) }
+  describe '#running' do
+    let(:q) { Queue.new }
+    let(:config) { common_config.merge('client_id' => 'test') }
+
+    before do
+      expect(subject).to receive(:create_consumer).once.and_return(consumer_double)
+      allow(consumer_double).to receive(:wakeup)
+      allow(consumer_double).to receive(:close)
+      allow(consumer_double).to receive(:subscribe)
+    end
 
-  it "should register" do
-    expect { subject.register }.to_not raise_error
+    context 'when running' do
+      before do
+        polled = false
+        allow(consumer_double).to receive(:poll) do
+          if polled
+            []
+          else
+            polled = true
+            payload
+          end
+        end
+
+        subject.register
+        t = Thread.new do
+          sleep(1)
+          subject.do_stop
+        end
+        subject.run(q)
+        t.join
+      end
+
+      it 'should process the correct number of events' do
+        expect(q.size).to eq(10)
+      end
+
+      it 'should set the consumer thread name' do
+        expect(subject.instance_variable_get('@runner_threads').first.get_name).to eq("kafka-input-worker-test-0")
+      end
+    end
+
+    context 'when errors are encountered during poll' do
+      before do
+        raised, polled = false
+        allow(consumer_double).to receive(:poll) do
+          unless raised
+            raised = true
+            raise exception
+          end
+          if polled
+            []
+          else
+            polled = true
+            payload
+          end
+        end
+
+        subject.register
+        t = Thread.new do
+          sleep 2
+          subject.do_stop
+        end
+        subject.run(q)
+        t.join
+      end
+
+      context "when a Kafka exception is raised" do
+        let(:exception) { org.apache.kafka.common.errors.TopicAuthorizationException.new('Invalid topic') }
+
+        it 'should poll successfully' do
+          expect(q.size).to eq(10)
+        end
+      end
+
+      context "when a StandardError is raised" do
+        let(:exception) { StandardError.new('Standard Error') }
+
+        it 'should retry and poll successfully' do
+          expect(q.size).to eq(10)
+        end
+      end
+
+      context "when a java error is raised" do
+        let(:exception) { java.lang.AssertionError.new('Fatal assertion') }
+
+        it "should not retry" do
+          expect(q.size).to eq(0)
+        end
+      end
+    end
   end
 
   context "register parameter verification" do
-    let(:config) do
-      { 'schema_registry_url' => 'http://localhost:8081', 'topics' => ['logstash'], 'consumer_threads' => 4 }
-    end
+    context "schema_registry_url" do
+      let(:config) do
+        { 'schema_registry_url' => 'http://localhost:8081', 'topics' => ['logstash'], 'consumer_threads' => 4 }
+      end
 
-    it "schema_registry_url conflict with value_deserializer_class should fail" do
-      config['value_deserializer_class'] = 'my.fantasy.Deserializer'
-      expect { subject.register }.to raise_error LogStash::ConfigurationError, /Option schema_registry_url prohibit the customization of value_deserializer_class/
+      it "conflict with value_deserializer_class should fail" do
+        config['value_deserializer_class'] = 'my.fantasy.Deserializer'
+        expect { subject.register }.to raise_error LogStash::ConfigurationError, /Option schema_registry_url prohibit the customization of value_deserializer_class/
+      end
+
+      it "conflict with topics_pattern should fail" do
+        config['topics_pattern'] = 'topic_.*'
+        expect { subject.register }.to raise_error LogStash::ConfigurationError, /Option schema_registry_url prohibit the customization of topics_pattern/
+      end
     end
 
-    it "schema_registry_url conflict with topics_pattern should fail" do
-      config['topics_pattern'] = 'topic_.*'
-      expect { subject.register }.to raise_error LogStash::ConfigurationError, /Option schema_registry_url prohibit the customization of topics_pattern/
+    context "decorate_events" do
+      let(:config) { { 'decorate_events' => 'extended'} }
+
+      it "should raise error for invalid value" do
+        config['decorate_events'] = 'avoid'
+        expect { subject.register }.to raise_error LogStash::ConfigurationError, /Something is wrong with your configuration./
+      end
+
+      it "should map old true boolean value to :record_props mode" do
+        config['decorate_events'] = "true"
+        subject.register
+        expect(subject.metadata_mode).to include(:record_props)
+      end
     end
   end
 
   context 'with client_rack' do
-    let(:config) { super.merge('client_rack' => 'EU-R1') }
+    let(:config) { super().merge('client_rack' => 'EU-R1') }
 
     it "sets broker rack parameter" do
       expect(org.apache.kafka.clients.consumer.KafkaConsumer).
@@ -66,7 +223,7 @@ describe LogStash::Inputs::Kafka do
   end
 
   context 'string integer config' do
-    let(:config) { super.merge('session_timeout_ms' => '25000', 'max_poll_interval_ms' => '345000') }
+    let(:config) { super().merge('session_timeout_ms' => '25000', 'max_poll_interval_ms' => '345000') }
 
     it "sets integer values" do
       expect(org.apache.kafka.clients.consumer.KafkaConsumer).
@@ -78,7 +235,7 @@ describe LogStash::Inputs::Kafka do
   end
 
   context 'integer config' do
-    let(:config) { super.merge('session_timeout_ms' => 25200, 'max_poll_interval_ms' => 123_000) }
+    let(:config) { super().merge('session_timeout_ms' => 25200, 'max_poll_interval_ms' => 123_000) }
 
     it "sets integer values" do
       expect(org.apache.kafka.clients.consumer.KafkaConsumer).
@@ -90,7 +247,7 @@ describe LogStash::Inputs::Kafka do
   end
 
   context 'string boolean config' do
-    let(:config) { super.merge('enable_auto_commit' => 'false', 'check_crcs' => 'true') }
+    let(:config) { super().merge('enable_auto_commit' => 'false', 'check_crcs' => 'true') }
 
     it "sets parameters" do
       expect(org.apache.kafka.clients.consumer.KafkaConsumer).
@@ -103,7 +260,7 @@ describe LogStash::Inputs::Kafka do
   end
 
   context 'boolean config' do
-    let(:config) { super.merge('enable_auto_commit' => true, 'check_crcs' => false) }
+    let(:config) { super().merge('enable_auto_commit' => true, 'check_crcs' => false) }
 
     it "sets parameters" do
       expect(org.apache.kafka.clients.consumer.KafkaConsumer).

data/spec/unit/outputs/kafka_spec.rb

@@ -8,6 +8,8 @@ describe "outputs/kafka" do
   let (:event) { LogStash::Event.new({'message' => 'hello', 'topic_name' => 'my_topic', 'host' => '172.0.0.1',
                                       '@timestamp' => LogStash::Timestamp.now}) }
 
+  let(:future) { double('kafka producer future') }
+
   context 'when initializing' do
     it "should register" do
       output = LogStash::Plugin.lookup("output", "kafka").new(simple_kafka_config)
@@ -24,8 +26,8 @@ describe "outputs/kafka" do
 
   context 'when outputting messages' do
     it 'should send logstash event to kafka broker' do
-      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
-        .with(an_instance_of(org.apache.kafka.clients.producer.ProducerRecord)).and_call_original
+      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).
+          with(an_instance_of(org.apache.kafka.clients.producer.ProducerRecord))
       kafka = LogStash::Outputs::Kafka.new(simple_kafka_config)
       kafka.register
       kafka.multi_receive([event])
@@ -33,18 +35,18 @@ describe "outputs/kafka" do
 
     it 'should support Event#sprintf placeholders in topic_id' do
       topic_field = 'topic_name'
-      expect(org.apache.kafka.clients.producer.ProducerRecord).to receive(:new)
-        .with("my_topic", event.to_s).and_call_original
-      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).and_call_original
+      expect(org.apache.kafka.clients.producer.ProducerRecord).to receive(:new).
+          with("my_topic", event.to_s).and_call_original
+      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
       kafka = LogStash::Outputs::Kafka.new({'topic_id' => "%{#{topic_field}}"})
       kafka.register
       kafka.multi_receive([event])
     end
 
     it 'should support field referenced message_keys' do
-      expect(org.apache.kafka.clients.producer.ProducerRecord).to receive(:new)
-        .with("test", "172.0.0.1", event.to_s).and_call_original
-      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).and_call_original
+      expect(org.apache.kafka.clients.producer.ProducerRecord).to receive(:new).
+          with("test", "172.0.0.1", event.to_s).and_call_original
+      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
       kafka = LogStash::Outputs::Kafka.new(simple_kafka_config.merge({"message_key" => "%{host}"}))
       kafka.register
       kafka.multi_receive([event])
@@ -71,22 +73,24 @@ describe "outputs/kafka" do
     before do
       count = 0
       expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
-        .exactly(sendcount).times
-        .and_wrap_original do |m, *args|
+        .exactly(sendcount).times do
         if count < failcount # fail 'failcount' times in a row.
           count += 1
           # Pick an exception at random
           raise exception_classes.shuffle.first.new("injected exception for testing")
         else
-          m.call(*args) # call original
+          count = :done
+          future # return future
         end
       end
+      expect(future).to receive :get
     end
 
     it "should retry until successful" do
       kafka = LogStash::Outputs::Kafka.new(simple_kafka_config)
       kafka.register
       kafka.multi_receive([event])
+      sleep(1.0) # allow for future.get call
     end
   end
 
@@ -101,15 +105,13 @@ describe "outputs/kafka" do
 
     before do
       count = 0
-      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
-        .exactly(1).times
-        .and_wrap_original do |m, *args|
+      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).exactly(1).times do
         if count < failcount # fail 'failcount' times in a row.
           count += 1
           # Pick an exception at random
           raise exception_classes.shuffle.first.new("injected exception for testing")
         else
-          m.call(*args) # call original
+          fail 'unexpected producer#send invocation'
         end
       end
     end
@@ -131,25 +133,24 @@ describe "outputs/kafka" do
 
       it "should retry until successful" do
         count = 0
-
-        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
-              .exactly(sendcount).times
-              .and_wrap_original do |m, *args|
+        success = nil
+        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).exactly(sendcount).times do
           if count < failcount
             count += 1
             # inject some failures.
 
             # Return a custom Future that will raise an exception to simulate a Kafka send() problem.
             future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
-            future.run
-            future
           else
-            m.call(*args)
+            success = true
+            future = java.util.concurrent.FutureTask.new { nil } # return no-op future
           end
+          future.tap { Thread.start { future.run } }
         end
         kafka = LogStash::Outputs::Kafka.new(simple_kafka_config)
         kafka.register
         kafka.multi_receive([event])
+        expect( success ).to be true
       end
     end
 
@@ -158,9 +159,7 @@ describe "outputs/kafka" do
       let(:max_sends) { 1 }
 
       it "should should only send once" do
-        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
-                                                                                       .once
-                                                                                       .and_wrap_original do |m, *args|
+        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).once do
           # Always fail.
           future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
           future.run
@@ -172,9 +171,7 @@ describe "outputs/kafka" do
       end
 
       it 'should not sleep' do
-        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
-                                                                                       .once
-                                                                                       .and_wrap_original do |m, *args|
+        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).once do
           # Always fail.
           future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
           future.run
@@ -193,13 +190,10 @@ describe "outputs/kafka" do
       let(:max_sends) { retries + 1 }
 
       it "should give up after retries are exhausted" do
-        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
-              .at_most(max_sends).times
-              .and_wrap_original do |m, *args|
+        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).at_most(max_sends).times do
           # Always fail.
           future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
-          future.run
-          future
+          future.tap { Thread.start { future.run } }
         end
         kafka = LogStash::Outputs::Kafka.new(simple_kafka_config.merge("retries" => retries))
         kafka.register
@@ -207,9 +201,7 @@ describe "outputs/kafka" do
       end
 
       it 'should only sleep retries number of times' do
-        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
-                                                                                       .at_most(max_sends).times
-                                                                                       .and_wrap_original do |m, *args|
+        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).at_most(max_sends).times do
           # Always fail.
           future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
           future.run

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-kafka
 version: !ruby/object:Gem::Version
-  version: 10.7.1
+  version: 10.7.6
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-11 00:00:00.000000000 Z
+date: 2021-05-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -126,6 +126,20 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 1.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-deprecation_logger_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -154,6 +168,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.1
+  name: digest-crc
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.1
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -206,6 +234,7 @@ files:
 - lib/logstash/plugin_mixins/common.rb
 - lib/logstash/plugin_mixins/kafka_support.rb
 - logstash-integration-kafka.gemspec
+- spec/check_docs_spec.rb
 - spec/fixtures/trust-store_stub.jks
 - spec/integration/inputs/kafka_spec.rb
 - spec/integration/outputs/kafka_spec.rb
@@ -222,7 +251,7 @@ files:
 - vendor/jar-dependencies/org/apache/avro/avro/1.9.2/avro-1.9.2.jar
 - vendor/jar-dependencies/org/apache/kafka/kafka-clients/2.5.1/kafka-clients-2.5.1.jar
 - vendor/jar-dependencies/org/apache/kafka/kafka_2.12/2.5.1/kafka_2.12-2.5.1.jar
-- vendor/jar-dependencies/org/glassfish/jersey/core/jersey-common/2.30/jersey-common-2.30.jar
+- vendor/jar-dependencies/org/glassfish/jersey/core/jersey-common/2.33/jersey-common-2.33.jar
 - vendor/jar-dependencies/org/lz4/lz4-java/1.7.1/lz4-java-1.7.1.jar
 - vendor/jar-dependencies/org/slf4j/slf4j-api/1.7.30/slf4j-api-1.7.30.jar
 - vendor/jar-dependencies/org/xerial/snappy/snappy-java/1.1.7.3/snappy-java-1.1.7.3.jar
@@ -255,6 +284,7 @@ signing_key:
 specification_version: 4
 summary: Integration with Kafka - input and output plugins
 test_files:
+- spec/check_docs_spec.rb
 - spec/fixtures/trust-store_stub.jks
 - spec/integration/inputs/kafka_spec.rb
 - spec/integration/outputs/kafka_spec.rb