logstash-integration-kafka 10.7.4-java → 10.8.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: efc6c33cf871ecd41fc07468d3d6e47dc1a71c4dbd1800fe99127da703547dd2
-  data.tar.gz: 644f506705807c95e15fc035aac7f5d57233dd5809ae37f2c71939324ab9c3e7
+  metadata.gz: a322e757007ef9c740f48a93ebe97aa50511a29b17450219ea4f2d08a49934e1
+  data.tar.gz: 111a1d72591d14876f33dba65cfc29e6c9ae397397f295e3a94e5078b413eb3a
 SHA512:
-  metadata.gz: 28341e37050a860c8e87d0b74a4da4d1fdd37eff3e9c95bd5596dff4d9a613149f4dd09052a949f6002fb6da835601df12ca6d37d03ea25819ff6517833da37c
-  data.tar.gz: eb196288b02dd30b92bf3a4e083be0b0d8f005307203e20e8892262d46e48041e4d4212182be685ecae668970530c7050c7132f4c2aaa3be50f3f00e46a04122
+  metadata.gz: bac61cfb956bb284df97746df68afc90b12b1d04186eba0cf27fc68886858dad965aefcf46566dc8e27e8a484b524c7ce8d3185d24d9412798e06defc9f78ab8
+  data.tar.gz: f706aaad143f31fdd31ea47c5d1604f490f505f08329ecd07dff75201c51d1983701e79e2f74a52f484c860eb026a26880447e3b89a985dffeb7367728cab1ef

data/CHANGELOG.md

@@ -1,3 +1,18 @@
+## 10.8.0
+  - Added config setting to enable schema registry validation to be skipped when an authentication scheme unsupported
+    by the validator is used [#97](https://github.com/logstash-plugins/logstash-integration-kafka/pull/97)
+
+## 10.7.7
+  - Fix: Correct the settings to allow basic auth to work properly, either by setting `schema_registry_key/secret` or embedding username/password in the
+    url [#94](https://github.com/logstash-plugins/logstash-integration-kafka/pull/94)
+
+## 10.7.6
+  - Test: specify development dependency version [#91](https://github.com/logstash-plugins/logstash-integration-kafka/pull/91)
+
+## 10.7.5
+  - Improved error handling in the input plugin to avoid errors 'escaping' from the plugin, and crashing the logstash
+    process [#87](https://github.com/logstash-plugins/logstash-integration-kafka/pull/87)
+
 ## 10.7.4
   - Docs: make sure Kafka clients version is updated in docs [#83](https://github.com/logstash-plugins/logstash-integration-kafka/pull/83)
     Since **10.6.0** Kafka client was updated to **2.5.1**

data/docs/input-kafka.asciidoc

@@ -128,6 +128,7 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai
 | <<plugins-{type}s-{plugin}-schema_registry_proxy>> |<<uri,uri>>|No
 | <<plugins-{type}s-{plugin}-schema_registry_secret>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-schema_registry_url>> |<<uri,uri>>|No
+| <<plugins-{type}s-{plugin}-schema_registry_validation>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-security_protocol>> |<<string,string>>, one of `["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"]`|No
 | <<plugins-{type}s-{plugin}-send_buffer_bytes>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-session_timeout_ms>> |<<number,number>>|No
@@ -576,6 +577,18 @@ The schemas must follow a naming convention with the pattern <topic name>-value.
 Use either the Schema Registry config option or the
 <<plugins-{type}s-{plugin}-value_deserializer_class>> config option, but not both.
 
+[id="plugins-{type}s-{plugin}-schema_registry_validation"]
+===== `schema_registry_validation`
+
+  * Value can be either of: `auto`, `skip`
+  * Default value is `"auto"`
+
+NOTE: Under most circumstances, the default setting of `auto` should not need to be changed.
+
+When using the schema registry, by default the plugin checks connectivity and validates the schema registry, during plugin registration, before events are processed.
+In some circumstances, this process may fail when it tries to validate an authenticated schema registry, causing the plugin to crash.
+This setting allows the plugin to skip validation during registration, which allows the plugin to continue and events to be processed. Note that an incorrectly configured schema registry will still stop the plugin from processing events.
+
 [id="plugins-{type}s-{plugin}-security_protocol"]
 ===== `security_protocol` 
 

data/lib/logstash/inputs/kafka.rb

@@ -253,6 +253,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   def register
     @runner_threads = []
     @metadata_mode = extract_metadata_level(@decorate_events)
+    @pattern ||= java.util.regex.Pattern.compile(@topics_pattern) unless @topics_pattern.nil?
     check_schema_registry_parameters
   end
 
@@ -280,9 +281,11 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
 
   public
   def run(logstash_queue)
-    @runner_consumers = consumer_threads.times.map { |i| create_consumer("#{client_id}-#{i}") }
-    @runner_threads = @runner_consumers.map { |consumer| thread_runner(logstash_queue, consumer) }
-    @runner_threads.each { |t| t.join }
+    @runner_consumers = consumer_threads.times.map { |i| subscribe(create_consumer("#{client_id}-#{i}")) }
+    @runner_threads = @runner_consumers.map.with_index { |consumer, i| thread_runner(logstash_queue, consumer,
+                                                                                     "kafka-input-worker-#{client_id}-#{i}") }
+    @runner_threads.each(&:start)
+    @runner_threads.each(&:join)
   end # def run
 
   public
@@ -296,62 +299,100 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
     @runner_consumers
   end
 
-  private
-  def thread_runner(logstash_queue, consumer)
-    Thread.new do
+  def subscribe(consumer)
+    @pattern.nil? ? consumer.subscribe(topics) : consumer.subscribe(@pattern)
+    consumer
+  end
+
+  def thread_runner(logstash_queue, consumer, name)
+    java.lang.Thread.new do
+      LogStash::Util::set_thread_name(name)
       begin
-        unless @topics_pattern.nil?
-          nooplistener = org.apache.kafka.clients.consumer.internals.NoOpConsumerRebalanceListener.new
-          pattern = java.util.regex.Pattern.compile(@topics_pattern)
-          consumer.subscribe(pattern, nooplistener)
-        else
-          consumer.subscribe(topics);
-        end
         codec_instance = @codec.clone
-        while !stop?
-          records = consumer.poll(poll_timeout_ms)
-          next unless records.count > 0
-          for record in records do
-            codec_instance.decode(record.value.to_s) do |event|
-              decorate(event)
-              if schema_registry_url
-                json = LogStash::Json.load(record.value.to_s)
-                json.each do |k, v|
-                  event.set(k, v)
-                end
-                event.remove("message")
-              end
-              if @metadata_mode.include?(:record_props)
-                event.set("[@metadata][kafka][topic]", record.topic)
-                event.set("[@metadata][kafka][consumer_group]", @group_id)
-                event.set("[@metadata][kafka][partition]", record.partition)
-                event.set("[@metadata][kafka][offset]", record.offset)
-                event.set("[@metadata][kafka][key]", record.key)
-                event.set("[@metadata][kafka][timestamp]", record.timestamp)
-              end
-              if @metadata_mode.include?(:headers)
-                record.headers.each do |header|
-                  s = String.from_java_bytes(header.value)
-                  s.force_encoding(Encoding::UTF_8)
-                  if s.valid_encoding?
-                    event.set("[@metadata][kafka][headers]["+header.key+"]", s)
-                  end
-                end
-              end
-              logstash_queue << event
-            end
+        until stop?
+          records = do_poll(consumer)
+          unless records.empty?
+            records.each { |record| handle_record(record, codec_instance, logstash_queue) }
+            maybe_commit_offset(consumer)
           end
-          # Manual offset commit
-          consumer.commitSync if @enable_auto_commit.eql?(false)
         end
-      rescue org.apache.kafka.common.errors.WakeupException => e
-        raise e if !stop?
       ensure
         consumer.close
       end
     end
   end
 
+  def do_poll(consumer)
+    records = []
+    begin
+      records = consumer.poll(poll_timeout_ms)
+    rescue org.apache.kafka.common.errors.WakeupException => e
+      logger.debug("Wake up from poll", :kafka_error_message => e)
+      raise e unless stop?
+    rescue => e
+      logger.error("Unable to poll Kafka consumer",
+                   :kafka_error_message => e,
+                   :cause => e.respond_to?(:getCause) ? e.getCause : nil)
+      Stud.stoppable_sleep(1) { stop? }
+    end
+    records
+  end
+
+  def handle_record(record, codec_instance, queue)
+    codec_instance.decode(record.value.to_s) do |event|
+      decorate(event)
+      maybe_apply_schema(event, record)
+      maybe_set_metadata(event, record)
+      queue << event
+    end
+  end
+
+  def maybe_apply_schema(event, record)
+    if schema_registry_url
+      json = LogStash::Json.load(record.value.to_s)
+      json.each do |k, v|
+        event.set(k, v)
+      end
+      event.remove("message")
+    end
+  end
+
+  def maybe_set_metadata(event, record)
+    if @metadata_mode.include?(:record_props)
+      event.set("[@metadata][kafka][topic]", record.topic)
+      event.set("[@metadata][kafka][consumer_group]", @group_id)
+      event.set("[@metadata][kafka][partition]", record.partition)
+      event.set("[@metadata][kafka][offset]", record.offset)
+      event.set("[@metadata][kafka][key]", record.key)
+      event.set("[@metadata][kafka][timestamp]", record.timestamp)
+    end
+    if @metadata_mode.include?(:headers)
+      record.headers.each do |header|
+        s = String.from_java_bytes(header.value)
+        s.force_encoding(Encoding::UTF_8)
+        if s.valid_encoding?
+          event.set("[@metadata][kafka][headers][" + header.key + "]", s)
+        end
+      end
+    end
+  end
+
+  def maybe_commit_offset(consumer)
+    begin
+      consumer.commitSync if @enable_auto_commit.eql?(false)
+    rescue org.apache.kafka.common.errors.WakeupException => e
+      logger.debug("Wake up from commitSync", :kafka_error_message => e)
+      raise e unless stop?
+    rescue StandardError => e
+      # For transient errors, the commit should be successful after the next set of
+      # polled records has been processed.
+      # But, it might also be worth thinking about adding a configurable retry mechanism
+      logger.error("Unable to commit records",
+                   :kafka_error_message => e,
+                   :cause => e.respond_to?(:getCause) ? e.getCause() : nil)
+    end
+  end
+
   private
   def create_consumer(client_id)
     begin
@@ -392,13 +433,16 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
       if schema_registry_url
         props.put(kafka::VALUE_DESERIALIZER_CLASS_CONFIG, Java::io.confluent.kafka.serializers.KafkaAvroDeserializer.java_class)
         serdes_config = Java::io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig
-        props.put(serdes_config::SCHEMA_REGISTRY_URL_CONFIG, schema_registry_url.to_s)
+        props.put(serdes_config::SCHEMA_REGISTRY_URL_CONFIG, schema_registry_url.uri.to_s)
         if schema_registry_proxy && !schema_registry_proxy.empty?
           props.put(serdes_config::PROXY_HOST, @schema_registry_proxy_host)
           props.put(serdes_config::PROXY_PORT, @schema_registry_proxy_port)
         end
         if schema_registry_key && !schema_registry_key.empty?
+          props.put(serdes_config::BASIC_AUTH_CREDENTIALS_SOURCE, 'USER_INFO')
           props.put(serdes_config::USER_INFO_CONFIG, schema_registry_key + ":" + schema_registry_secret.value)
+        else
+          props.put(serdes_config::BASIC_AUTH_CREDENTIALS_SOURCE, 'URL')
         end
       end
       if security_protocol == "SSL"

data/lib/logstash/plugin_mixins/common.rb

@@ -22,6 +22,10 @@ module LogStash
         # Option to set the proxy of the Schema Registry.
         # This option permits to define a proxy to be used to reach the schema registry service instance.
         config :schema_registry_proxy, :validate => :uri
+
+        # Option to skip validating the schema registry during registration. This can be useful when using
+        # certificate based auth
+        config :schema_registry_validation, :validate => ['auto', 'skip'], :default => 'auto'
       end
 
       def check_schema_registry_parameters
@@ -29,10 +33,21 @@ module LogStash
           check_for_schema_registry_conflicts
           @schema_registry_proxy_host, @schema_registry_proxy_port  = split_proxy_into_host_and_port(schema_registry_proxy)
           check_for_key_and_secret
-          check_for_schema_registry_connectivity_and_subjects
+          check_for_schema_registry_connectivity_and_subjects if schema_registry_validation?
         end
       end
 
+      def schema_registry_validation?
+        return false if schema_registry_validation.to_s == 'skip'
+        return false if using_kerberos? # pre-validation doesn't support kerberos
+        
+        true
+      end
+
+      def using_kerberos?
+        security_protocol == "SASL_PLAINTEXT" || security_protocol == "SASL_SSL"
+      end
+
       private
       def check_for_schema_registry_conflicts
         if @value_deserializer_class != LogStash::Inputs::Kafka::DEFAULT_DESERIALIZER_CLASS
@@ -53,9 +68,8 @@ module LogStash
           options[:auth] = {:user => schema_registry_key, :password => schema_registry_secret.value}
         end
         client = Manticore::Client.new(options)
-
         begin
-          response = client.get(@schema_registry_url.to_s + '/subjects').body
+          response = client.get(@schema_registry_url.uri.to_s + '/subjects').body
         rescue Manticore::ManticoreException => e
           raise LogStash::ConfigurationError.new("Schema registry service doesn't respond, error: #{e.message}")
         end

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '10.7.4'
+  s.version         = '10.8.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+
@@ -51,6 +51,7 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'rspec-wait'
-  s.add_development_dependency 'ruby-kafka'
+  s.add_development_dependency 'digest-crc', '~> 0.5.1' # 0.6.0 started using a C-ext
+  s.add_development_dependency 'ruby-kafka' # depends on digest-crc
   s.add_development_dependency 'snappy'
 end

data/spec/check_docs_spec.rb

@@ -0,0 +1,36 @@
+# encoding: utf-8
+require 'logstash-integration-kafka_jars'
+
+describe "[DOCS]" do
+
+  let(:docs_files) do
+    ['index.asciidoc', 'input-kafka.asciidoc', 'output-kafka.asciidoc'].map { |name| File.join('docs', name) }
+  end
+
+  let(:kafka_version_properties) do
+    loader = java.lang.Thread.currentThread.getContextClassLoader
+    version = loader.getResource('kafka/kafka-version.properties')
+    fail "kafka-version.properties missing" unless version
+    properties = java.util.Properties.new
+    properties.load version.openStream
+    properties
+  end
+
+  it 'is sync-ed with Kafka client version' do
+    version = kafka_version_properties.get('version') # e.g. '2.5.1'
+
+    fails = docs_files.map do |file|
+      if line = File.readlines(file).find { |line| line.index(':kafka_client:') }
+        puts "found #{line.inspect} in #{file}" if $VERBOSE # e.g. ":kafka_client: 2.5\n"
+        if !version.start_with?(line.strip.split[1])
+          "documentation at #{file} is out of sync with kafka-clients version (#{version.inspect}), detected line: #{line.inspect}"
+        else
+          nil
+        end
+      end
+    end
+
+    fail "\n" + fails.join("\n") if fails.flatten.any?
+  end
+
+end

data/spec/fixtures/jaas.config

@@ -0,0 +1,5 @@
+SchemaRegistry-Props {
+  org.eclipse.jetty.jaas.spi.PropertyFileLoginModule required
+  file="build/confluent_platform/etc/schema-registry/pwd"
+  debug="true";
+};

data/spec/fixtures/pwd

@@ -0,0 +1,5 @@
+fred: OBF:1w8t1tvf1w261w8v1w1c1tvn1w8x,user,admin
+barney: changeme,user,developer
+admin:admin,admin
+betty: MD5:164c88b302622e17050af52c89945d44,user
+wilma: CRYPT:adpexzg3FUZAk,admin,sr-user

data/spec/integration/inputs/kafka_spec.rb

@@ -198,13 +198,24 @@ def consume_messages(config, queue: Queue.new, timeout:, event_count:)
     wait(timeout).for { queue.length }.to eq(event_count) unless timeout.eql?(false)
     block_given? ? yield(queue, kafka_input) : queue
   ensure
+    kafka_input.do_stop
     t.kill
-    t.join(30_000)
+    t.join(30)
   end
 end
 
 
 describe "schema registry connection options" do
+  schema_registry = Manticore::Client.new
+  before (:all) do
+    shutdown_schema_registry
+    startup_schema_registry(schema_registry)
+  end
+
+  after(:all) do
+    shutdown_schema_registry
+  end
+
   context "remote endpoint validation" do
     it "should fail if not reachable" do
       config = {'schema_registry_url' => 'http://localnothost:8081'}
@@ -231,8 +242,7 @@ describe "schema registry connection options" do
       end
 
       after(:each) do
-        schema_registry_client = Manticore::Client.new
-        delete_remote_schema(schema_registry_client, SUBJECT_NAME)
+        delete_remote_schema(schema_registry, SUBJECT_NAME)
       end
 
       it "should correctly complete registration phase" do
@@ -263,9 +273,25 @@ end
 
 # AdminClientConfig = org.alpache.kafka.clients.admin.AdminClientConfig
 
+def startup_schema_registry(schema_registry, auth=false)
+  system('./stop_schema_registry.sh')
+  auth ? system('./start_auth_schema_registry.sh') : system('./start_schema_registry.sh')
+  url = auth ? "http://barney:changeme@localhost:8081" : "http://localhost:8081"
+  Stud.try(20.times, [Manticore::SocketException, StandardError, RSpec::Expectations::ExpectationNotMetError]) do
+    expect(schema_registry.get(url).code).to eq(200)
+  end
+end
+
 describe "Schema registry API", :integration => true do
+  schema_registry = Manticore::Client.new
+
+  before(:all) do
+    startup_schema_registry(schema_registry)
+  end
 
-  let(:schema_registry) { Manticore::Client.new }
+  after(:all) do
+    shutdown_schema_registry
+  end
 
   context 'listing subject on clean instance' do
     it "should return an empty set" do
@@ -291,37 +317,58 @@ describe "Schema registry API", :integration => true do
       expect( subjects ).to be_empty
     end
   end
+end
+
+def shutdown_schema_registry
+  system('./stop_schema_registry.sh')
+end
+
+describe "Deserializing with the schema registry", :integration => true do
+  schema_registry = Manticore::Client.new
+
+  shared_examples 'it reads from a topic using a schema registry' do |with_auth|
+
+    before(:all) do
+      shutdown_schema_registry
+      startup_schema_registry(schema_registry, with_auth)
+    end
+
+    after(:all) do
+      shutdown_schema_registry
+    end
 
-  context 'use the schema to serialize' do
     after(:each) do
-      expect( schema_registry.delete('http://localhost:8081/subjects/topic_avro-value').code ).to be(200)
+      expect( schema_registry.delete("#{subject_url}/#{avro_topic_name}-value").code ).to be(200)
       sleep 1
-      expect( schema_registry.delete('http://localhost:8081/subjects/topic_avro-value?permanent=true').code ).to be(200)
+      expect( schema_registry.delete("#{subject_url}/#{avro_topic_name}-value?permanent=true").code ).to be(200)
 
       Stud.try(3.times, [StandardError, RSpec::Expectations::ExpectationNotMetError]) do
         wait(10).for do
-          subjects = JSON.parse schema_registry.get('http://localhost:8081/subjects').body
+          subjects = JSON.parse schema_registry.get(subject_url).body
           subjects.empty?
         end.to be_truthy
       end
     end
 
-    let(:group_id_1) {rand(36**8).to_s(36)}
-
-    let(:avro_topic_name) { "topic_avro" }
-
-    let(:plain_config) do
-      { 'schema_registry_url' => 'http://localhost:8081',
-        'topics' => [avro_topic_name],
-        'codec' => 'plain',
-        'group_id' => group_id_1,
-        'auto_offset_reset' => 'earliest' }
+    let(:base_config) do
+      {
+          'topics' => [avro_topic_name],
+          'codec' => 'plain',
+          'group_id' => group_id_1,
+          'auto_offset_reset' => 'earliest'
+      }
     end
 
-    def delete_topic_if_exists(topic_name)
+    let(:group_id_1) {rand(36**8).to_s(36)}
+
+    def delete_topic_if_exists(topic_name, user = nil, password = nil)
       props = java.util.Properties.new
       props.put(Java::org.apache.kafka.clients.admin.AdminClientConfig::BOOTSTRAP_SERVERS_CONFIG, "localhost:9092")
-
+      serdes_config = Java::io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig
+      unless user.nil?
+        props.put(serdes_config::BASIC_AUTH_CREDENTIALS_SOURCE, 'USER_INFO')
+        props.put(serdes_config::USER_INFO_CONFIG,  "#{user}:#{password}")
+      end
       admin_client = org.apache.kafka.clients.admin.AdminClient.create(props)
       topics_list = admin_client.listTopics().names().get()
       if topics_list.contains(topic_name)
@@ -330,7 +377,7 @@ describe "Schema registry API", :integration => true do
       end
     end
 
-    def write_some_data_to(topic_name)
+    def write_some_data_to(topic_name, user = nil, password = nil)
       props = java.util.Properties.new
       config = org.apache.kafka.clients.producer.ProducerConfig
 
@@ -338,6 +385,10 @@ describe "Schema registry API", :integration => true do
       props.put(serdes_config::SCHEMA_REGISTRY_URL_CONFIG, "http://localhost:8081")
 
       props.put(config::BOOTSTRAP_SERVERS_CONFIG, "localhost:9092")
+      unless user.nil?
+        props.put(serdes_config::BASIC_AUTH_CREDENTIALS_SOURCE, 'USER_INFO')
+        props.put(serdes_config::USER_INFO_CONFIG,  "#{user}:#{password}")
+      end
       props.put(config::KEY_SERIALIZER_CLASS_CONFIG, org.apache.kafka.common.serialization.StringSerializer.java_class)
       props.put(config::VALUE_SERIALIZER_CLASS_CONFIG, Java::io.confluent.kafka.serializers.KafkaAvroSerializer.java_class)
 
@@ -359,11 +410,11 @@ describe "Schema registry API", :integration => true do
     end
 
     it "stored a new schema using Avro Kafka serdes" do
-      delete_topic_if_exists avro_topic_name
-      write_some_data_to avro_topic_name
+      auth ? delete_topic_if_exists(avro_topic_name, user, password) : delete_topic_if_exists(avro_topic_name)
+      auth ? write_some_data_to(avro_topic_name, user, password) : write_some_data_to(avro_topic_name)
 
-      subjects = JSON.parse schema_registry.get('http://localhost:8081/subjects').body
-      expect( subjects ).to contain_exactly("topic_avro-value")
+      subjects = JSON.parse schema_registry.get(subject_url).body
+      expect( subjects ).to contain_exactly("#{avro_topic_name}-value")
 
       num_events = 1
       queue = consume_messages(plain_config, timeout: 30, event_count: num_events)
@@ -374,4 +425,43 @@ describe "Schema registry API", :integration => true do
       expect( elem.get("map_field")["inner_field"] ).to eq("inner value")
     end
   end
+
+  context 'with an unauthed schema registry' do
+    let(:auth) { false }
+    let(:avro_topic_name) { "topic_avro" }
+    let(:subject_url) { "http://localhost:8081/subjects" }
+    let(:plain_config)  { base_config.merge!({'schema_registry_url' => "http://localhost:8081"}) }
+
+    it_behaves_like 'it reads from a topic using a schema registry', false
+  end
+
+  context 'with an authed schema registry' do
+    let(:auth) { true }
+    let(:user) { "barney" }
+    let(:password) { "changeme" }
+    let(:avro_topic_name) { "topic_avro_auth" }
+    let(:subject_url) { "http://#{user}:#{password}@localhost:8081/subjects" }
+
+    context 'using schema_registry_key' do
+      let(:plain_config) do
+        base_config.merge!({
+          'schema_registry_url' => "http://localhost:8081",
+          'schema_registry_key' => user,
+          'schema_registry_secret' => password
+        })
+      end
+
+      it_behaves_like 'it reads from a topic using a schema registry', true
+    end
+
+    context 'using schema_registry_url' do
+      let(:plain_config) do
+        base_config.merge!({
+          'schema_registry_url' => "http://#{user}:#{password}@localhost:8081"
+        })
+      end
+
+      it_behaves_like 'it reads from a topic using a schema registry', true
+    end
+  end
 end

data/spec/integration/outputs/kafka_spec.rb

@@ -44,7 +44,7 @@ describe "outputs/kafka", :integration => true do
   end
 
   context 'when outputting messages serialized as Byte Array' do
-    let(:test_topic) { 'topic1b' }
+    let(:test_topic) { 'logstash_integration_topicbytearray' }
     let(:num_events) { 3 }
 
     before :each do

data/spec/unit/inputs/kafka_spec.rb

@@ -3,45 +3,190 @@ require "logstash/devutils/rspec/spec_helper"
 require "logstash/inputs/kafka"
 require "concurrent"
 
-class MockConsumer
-  def initialize
-    @wake = Concurrent::AtomicBoolean.new(false)
-  end
 
-  def subscribe(topics)
-  end
-  
-  def poll(ms)
-    if @wake.value
-      raise org.apache.kafka.common.errors.WakeupException.new
-    else
-      10.times.map do
-        org.apache.kafka.clients.consumer.ConsumerRecord.new("logstash", 0, 0, "key", "value")
+describe LogStash::Inputs::Kafka do
+  let(:common_config) { { 'topics' => ['logstash'] } }
+  let(:config) { common_config }
+  let(:consumer_double) { double(:consumer) }
+  let(:needs_raise) { false }
+  let(:payload) {
+    10.times.map do
+      org.apache.kafka.clients.consumer.ConsumerRecord.new("logstash", 0, 0, "key", "value")
+    end
+  }
+  subject { LogStash::Inputs::Kafka.new(config) }
+
+  describe '#poll' do
+    before do
+      polled = false
+      allow(consumer_double).to receive(:poll) do
+        if polled
+          []
+        else
+          polled = true
+          payload
+        end
       end
     end
+
+    it 'should poll' do
+      expect(consumer_double).to receive(:poll)
+      expect(subject.do_poll(consumer_double)).to eq(payload)
+    end
+
+    it 'should return nil if Kafka Exception is encountered' do
+      expect(consumer_double).to receive(:poll).and_raise(org.apache.kafka.common.errors.TopicAuthorizationException.new(''))
+      expect(subject.do_poll(consumer_double)).to be_empty
+    end
+
+    it 'should not throw if Kafka Exception is encountered' do
+      expect(consumer_double).to receive(:poll).and_raise(org.apache.kafka.common.errors.TopicAuthorizationException.new(''))
+      expect{subject.do_poll(consumer_double)}.not_to raise_error
+    end
+
+    it 'should return no records if Assertion Error is encountered' do
+      expect(consumer_double).to receive(:poll).and_raise(java.lang.AssertionError.new(''))
+      expect{subject.do_poll(consumer_double)}.to raise_error(java.lang.AssertionError)
+    end
   end
 
-  def close
+  describe '#maybe_commit_offset' do
+    context 'with auto commit disabled' do
+      let(:config) { common_config.merge('enable_auto_commit' => false) }
+
+      it 'should call commit on the consumer' do
+        expect(consumer_double).to receive(:commitSync)
+        subject.maybe_commit_offset(consumer_double)
+      end
+      it 'should not throw if a Kafka Exception is encountered' do
+        expect(consumer_double).to receive(:commitSync).and_raise(org.apache.kafka.common.errors.TopicAuthorizationException.new(''))
+        expect{subject.maybe_commit_offset(consumer_double)}.not_to raise_error
+      end
+
+      it 'should throw if Assertion Error is encountered' do
+        expect(consumer_double).to receive(:commitSync).and_raise(java.lang.AssertionError.new(''))
+        expect{subject.maybe_commit_offset(consumer_double)}.to raise_error(java.lang.AssertionError)
+      end
+    end
+
+    context 'with auto commit enabled' do
+      let(:config) { common_config.merge('enable_auto_commit' => true) }
+
+      it 'should not call commit on the consumer' do
+        expect(consumer_double).not_to receive(:commitSync)
+        subject.maybe_commit_offset(consumer_double)
+      end
+    end
   end
 
-  def wakeup
-    @wake.make_true
+  describe '#register' do
+    it "should register" do
+      expect { subject.register }.to_not raise_error
+    end
   end
-end
 
-describe LogStash::Inputs::Kafka do
-  let(:config) { { 'topics' => ['logstash'], 'consumer_threads' => 4 } }
-  subject { LogStash::Inputs::Kafka.new(config) }
+  describe '#running' do
+    let(:q) { Queue.new }
+    let(:config) { common_config.merge('client_id' => 'test') }
+
+    before do
+      expect(subject).to receive(:create_consumer).once.and_return(consumer_double)
+      allow(consumer_double).to receive(:wakeup)
+      allow(consumer_double).to receive(:close)
+      allow(consumer_double).to receive(:subscribe)
+    end
+
+    context 'when running' do
+      before do
+        polled = false
+        allow(consumer_double).to receive(:poll) do
+          if polled
+            []
+          else
+            polled = true
+            payload
+          end
+        end
+
+        subject.register
+        t = Thread.new do
+          sleep(1)
+          subject.do_stop
+        end
+        subject.run(q)
+        t.join
+      end
+
+      it 'should process the correct number of events' do
+        expect(q.size).to eq(10)
+      end
+
+      it 'should set the consumer thread name' do
+        expect(subject.instance_variable_get('@runner_threads').first.get_name).to eq("kafka-input-worker-test-0")
+      end
+    end
+
+    context 'when errors are encountered during poll' do
+      before do
+        raised, polled = false
+        allow(consumer_double).to receive(:poll) do
+          unless raised
+            raised = true
+            raise exception
+          end
+          if polled
+            []
+          else
+            polled = true
+            payload
+          end
+        end
+
+        subject.register
+        t = Thread.new do
+          sleep 2
+          subject.do_stop
+        end
+        subject.run(q)
+        t.join
+      end
+
+      context "when a Kafka exception is raised" do
+        let(:exception) { org.apache.kafka.common.errors.TopicAuthorizationException.new('Invalid topic') }
+
+        it 'should poll successfully' do
+          expect(q.size).to eq(10)
+        end
+      end
+
+      context "when a StandardError is raised" do
+        let(:exception) { StandardError.new('Standard Error') }
+
+        it 'should retry and poll successfully' do
+          expect(q.size).to eq(10)
+        end
+      end
+
+      context "when a java error is raised" do
+        let(:exception) { java.lang.AssertionError.new('Fatal assertion') }
 
-  it "should register" do
-    expect { subject.register }.to_not raise_error
+        it "should not retry" do
+          expect(q.size).to eq(0)
+        end
+      end
+    end
   end
 
-  context "register parameter verification" do
+  describe "schema registry parameter verification" do
+    let(:base_config) do {
+          'schema_registry_url' => 'http://localhost:8081',
+          'topics' => ['logstash'],
+          'consumer_threads' => 4
+    }
+    end
+
     context "schema_registry_url" do
-      let(:config) do
-        { 'schema_registry_url' => 'http://localhost:8081', 'topics' => ['logstash'], 'consumer_threads' => 4 }
-      end
+     let(:config) { base_config }
 
       it "conflict with value_deserializer_class should fail" do
         config['value_deserializer_class'] = 'my.fantasy.Deserializer'
@@ -54,19 +199,63 @@ describe LogStash::Inputs::Kafka do
       end
     end
 
-    context "decorate_events" do
-      let(:config) { { 'decorate_events' => 'extended'} }
+    context 'when kerberos auth is used' do
+      ['SASL_SSL', 'SASL_PLAINTEXT'].each do |protocol|
+        context "with #{protocol}" do
+          ['auto', 'skip'].each do |vsr|
+            context "when validata_schema_registry is #{vsr}" do
+              let(:config) { base_config.merge({'security_protocol' => protocol,
+                                                'schema_registry_validation' => vsr})
+              }
+              it 'skips verification' do
+                expect(subject).not_to receive(:check_for_schema_registry_connectivity_and_subjects)
+                expect { subject.register }.not_to raise_error
+              end
+            end
+          end
+        end
+      end
+    end
 
-      it "should raise error for invalid value" do
-        config['decorate_events'] = 'avoid'
-        expect { subject.register }.to raise_error LogStash::ConfigurationError, /Something is wrong with your configuration./
+    context 'when kerberos auth is not used' do
+      context "when skip_verify is set to auto" do
+        let(:config) { base_config.merge({'schema_registry_validation' => 'auto'})}
+        it 'performs verification' do
+          expect(subject).to receive(:check_for_schema_registry_connectivity_and_subjects)
+          expect { subject.register }.not_to raise_error
+        end
       end
 
-      it "should map old true boolean value to :record_props mode" do
-        config['decorate_events'] = "true"
-        subject.register
-        expect(subject.metadata_mode).to include(:record_props)
+      context "when skip_verify is set to default" do
+        let(:config) { base_config }
+        it 'performs verification' do
+          expect(subject).to receive(:check_for_schema_registry_connectivity_and_subjects)
+          expect { subject.register }.not_to raise_error
+        end
       end
+
+      context "when skip_verify is set to skip" do
+        let(:config) { base_config.merge({'schema_registry_validation' => 'skip'})}
+        it 'should skip verification' do
+          expect(subject).not_to receive(:check_for_schema_registry_connectivity_and_subjects)
+          expect { subject.register }.not_to raise_error
+        end
+      end
+    end
+  end
+
+  context "decorate_events" do
+    let(:config) { { 'decorate_events' => 'extended'} }
+
+    it "should raise error for invalid value" do
+      config['decorate_events'] = 'avoid'
+      expect { subject.register }.to raise_error LogStash::ConfigurationError, /Something is wrong with your configuration./
+    end
+
+    it "should map old true boolean value to :record_props mode" do
+      config['decorate_events'] = "true"
+      subject.register
+      expect(subject.metadata_mode).to include(:record_props)
     end
   end
 

data/spec/unit/outputs/kafka_spec.rb

@@ -8,6 +8,8 @@ describe "outputs/kafka" do
   let (:event) { LogStash::Event.new({'message' => 'hello', 'topic_name' => 'my_topic', 'host' => '172.0.0.1',
                                       '@timestamp' => LogStash::Timestamp.now}) }
 
+  let(:future) { double('kafka producer future') }
+
   context 'when initializing' do
     it "should register" do
       output = LogStash::Plugin.lookup("output", "kafka").new(simple_kafka_config)
@@ -24,8 +26,8 @@ describe "outputs/kafka" do
 
   context 'when outputting messages' do
     it 'should send logstash event to kafka broker' do
-      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
-        .with(an_instance_of(org.apache.kafka.clients.producer.ProducerRecord)).and_call_original
+      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).
+          with(an_instance_of(org.apache.kafka.clients.producer.ProducerRecord))
       kafka = LogStash::Outputs::Kafka.new(simple_kafka_config)
       kafka.register
       kafka.multi_receive([event])
@@ -33,18 +35,18 @@ describe "outputs/kafka" do
 
     it 'should support Event#sprintf placeholders in topic_id' do
       topic_field = 'topic_name'
-      expect(org.apache.kafka.clients.producer.ProducerRecord).to receive(:new)
-        .with("my_topic", event.to_s).and_call_original
-      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).and_call_original
+      expect(org.apache.kafka.clients.producer.ProducerRecord).to receive(:new).
+          with("my_topic", event.to_s).and_call_original
+      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
       kafka = LogStash::Outputs::Kafka.new({'topic_id' => "%{#{topic_field}}"})
       kafka.register
       kafka.multi_receive([event])
     end
 
     it 'should support field referenced message_keys' do
-      expect(org.apache.kafka.clients.producer.ProducerRecord).to receive(:new)
-        .with("test", "172.0.0.1", event.to_s).and_call_original
-      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).and_call_original
+      expect(org.apache.kafka.clients.producer.ProducerRecord).to receive(:new).
+          with("test", "172.0.0.1", event.to_s).and_call_original
+      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
       kafka = LogStash::Outputs::Kafka.new(simple_kafka_config.merge({"message_key" => "%{host}"}))
       kafka.register
       kafka.multi_receive([event])
@@ -71,22 +73,24 @@ describe "outputs/kafka" do
     before do
       count = 0
       expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
-        .exactly(sendcount).times
-        .and_wrap_original do |m, *args|
+        .exactly(sendcount).times do
         if count < failcount # fail 'failcount' times in a row.
           count += 1
           # Pick an exception at random
           raise exception_classes.shuffle.first.new("injected exception for testing")
         else
-          m.call(*args) # call original
+          count = :done
+          future # return future
         end
       end
+      expect(future).to receive :get
     end
 
     it "should retry until successful" do
       kafka = LogStash::Outputs::Kafka.new(simple_kafka_config)
       kafka.register
       kafka.multi_receive([event])
+      sleep(1.0) # allow for future.get call
     end
   end
 
@@ -101,15 +105,13 @@ describe "outputs/kafka" do
 
     before do
       count = 0
-      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
-        .exactly(1).times
-        .and_wrap_original do |m, *args|
+      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).exactly(1).times do
         if count < failcount # fail 'failcount' times in a row.
           count += 1
           # Pick an exception at random
           raise exception_classes.shuffle.first.new("injected exception for testing")
         else
-          m.call(*args) # call original
+          fail 'unexpected producer#send invocation'
         end
       end
     end
@@ -131,25 +133,24 @@ describe "outputs/kafka" do
 
       it "should retry until successful" do
         count = 0
-
-        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
-              .exactly(sendcount).times
-              .and_wrap_original do |m, *args|
+        success = nil
+        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).exactly(sendcount).times do
           if count < failcount
             count += 1
             # inject some failures.
 
             # Return a custom Future that will raise an exception to simulate a Kafka send() problem.
             future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
-            future.run
-            future
           else
-            m.call(*args)
+            success = true
+            future = java.util.concurrent.FutureTask.new { nil } # return no-op future
           end
+          future.tap { Thread.start { future.run } }
         end
         kafka = LogStash::Outputs::Kafka.new(simple_kafka_config)
         kafka.register
         kafka.multi_receive([event])
+        expect( success ).to be true
       end
     end
 
@@ -158,9 +159,7 @@ describe "outputs/kafka" do
       let(:max_sends) { 1 }
 
       it "should should only send once" do
-        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
-                                                                                       .once
-                                                                                       .and_wrap_original do |m, *args|
+        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).once do
           # Always fail.
           future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
           future.run
@@ -172,9 +171,7 @@ describe "outputs/kafka" do
       end
 
       it 'should not sleep' do
-        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
-                                                                                       .once
-                                                                                       .and_wrap_original do |m, *args|
+        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).once do
           # Always fail.
           future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
           future.run
@@ -193,13 +190,10 @@ describe "outputs/kafka" do
       let(:max_sends) { retries + 1 }
 
       it "should give up after retries are exhausted" do
-        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
-              .at_most(max_sends).times
-              .and_wrap_original do |m, *args|
+        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).at_most(max_sends).times do
           # Always fail.
           future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
-          future.run
-          future
+          future.tap { Thread.start { future.run } }
         end
         kafka = LogStash::Outputs::Kafka.new(simple_kafka_config.merge("retries" => retries))
         kafka.register
@@ -207,9 +201,7 @@ describe "outputs/kafka" do
       end
 
       it 'should only sleep retries number of times' do
-        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
-                                                                                       .at_most(max_sends).times
-                                                                                       .and_wrap_original do |m, *args|
+        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).at_most(max_sends).times do
           # Always fail.
           future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
           future.run

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-kafka
 version: !ruby/object:Gem::Version
-  version: 10.7.4
+  version: 10.8.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-14 00:00:00.000000000 Z
+date: 2021-07-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -168,6 +168,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.1
+  name: digest-crc
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.1
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -220,6 +234,9 @@ files:
 - lib/logstash/plugin_mixins/common.rb
 - lib/logstash/plugin_mixins/kafka_support.rb
 - logstash-integration-kafka.gemspec
+- spec/check_docs_spec.rb
+- spec/fixtures/jaas.config
+- spec/fixtures/pwd
 - spec/fixtures/trust-store_stub.jks
 - spec/integration/inputs/kafka_spec.rb
 - spec/integration/outputs/kafka_spec.rb
@@ -269,6 +286,9 @@ signing_key:
 specification_version: 4
 summary: Integration with Kafka - input and output plugins
 test_files:
+- spec/check_docs_spec.rb
+- spec/fixtures/jaas.config
+- spec/fixtures/pwd
 - spec/fixtures/trust-store_stub.jks
 - spec/integration/inputs/kafka_spec.rb
 - spec/integration/outputs/kafka_spec.rb