logstash-output-kafka 5.0.4 → 5.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2efe7b4ae424f118b3400de62e7784929fd3f136
-  data.tar.gz: 5c2f3136a44c69802972ef17014ccaf5adc5f868
+  metadata.gz: e9279e067fa0adfbc321675a10328320ef8f50d8
+  data.tar.gz: 6d1b7bf832ca676cfde01fca475f65a60f2ae33d
 SHA512:
-  metadata.gz: ac60bebd7d56ba7f6cffaa70c12ec369036882de6e4ecae2662cc5f7318f02b241914628dc3ed8abfdccb8430d34c5d4f8f2d633cf2da600dbef057f7fcbb7ab
-  data.tar.gz: 6e10ced8df8376178d6f3031fa8157565f823ce1850c5a6aae045b99eb78dea7156aef3613ad5ed8d75247128905d23fbf504f4667d909a23d21915a10a5fa01
+  metadata.gz: 10cb4458e38f2b49e48a2b5886a6a0751a1161a8ebdda893e80de8de38d2cef6036f50d15f8b40f7385f39b248373036628952878125edc38c58a630ba615726
+  data.tar.gz: 51e40dae2b3450ddf3d896bdcfe104f73553ef812cd3e397aceb90bcecde3764d93547bb6d0d4acd52ecbd0179954da24f950ee95858033cd0f674f621a6ddea

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 5.1.0
+  - Add Kerberos authentication feature.
+
+## 5.0.5
+  - Fix logging
+
 ## 5.0.4
   - Update to 0.10.0.1
 

data/lib/logstash/outputs/kafka.rb

@@ -6,20 +6,21 @@ require 'logstash-output-kafka_jars.rb'
 # Write events to a Kafka topic. This uses the Kafka Producer API to write messages to a topic on
 # the broker.
 #
-# Here's a compatibility matrix that shows the Kafka broker and client versions that are compatible with each combination
+# Here's a compatibility matrix that shows the Kafka client versions that are compatible with each combination
 # of Logstash and the Kafka output plugin: 
 # 
 # [options="header"]
 # |==========================================================
-# |Kafka Broker Version |Kafka Client Version |Logstash Version |Plugin Version |Why?
-# |0.8       |0.8       |2.0.0 - 2.x.x   |<3.0.0 |Legacy, 0.8 is still popular 
-# |0.9       |0.9       |2.0.0 - 2.3.x   | 3.x.x |Works with the old Ruby Event API (`event['product']['price'] = 10`)  
-# |0.9       |0.9       |2.4.0 - 5.0.x   | 4.x.x |Works with the new getter/setter APIs (`event.set('[product][price]', 10)`)
-# |0.10      |0.10      |2.4.0 - 5.0.x   | 5.x.x |Not compatible with the 0.9 broker 
+# |Kafka Client Version |Logstash Version |Plugin Version |Security Features |Why?
+# |0.8       |2.0.0 - 2.x.x   |<3.0.0 | |Legacy, 0.8 is still popular 
+# |0.9       |2.0.0 - 2.3.x   | 3.x.x |Basic Auth, SSL |Works with the old Ruby Event API (`event['product']['price'] = 10`)  
+# |0.9       |2.4.0 - 5.0.x   | 4.x.x |Basic Auth, SSL |Works with the new getter/setter APIs (`event.set('[product][price]', 10)`)
+# |0.10      |2.4.0 - 5.0.x   | 5.x.x |Basic Auth, SSL |Not compatible with the 0.9 broker 
 # |==========================================================
 # 
-# NOTE: It's a good idea to upgrade brokers before consumers/producers because brokers target backwards compatibility.
-# For example, the 0.9 broker will work with both the 0.8 consumer and 0.9 consumer APIs, but not the other way around.
+# NOTE: We recommended that you use matching Kafka client and broker versions. During upgrades, you should
+# upgrade brokers before clients because brokers target backwards compatibility. For example, the 0.9 broker
+# is compatible with both the 0.8 consumer and 0.9 consumer APIs, but not the other way around.
 #
 # The only required configuration is the topic_id. The default codec is json,
 # so events will be persisted on the broker in json format. If you select a codec of plain,
@@ -111,15 +112,49 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   # The size of the TCP send buffer to use when sending data.
   config :send_buffer_bytes, :validate => :number, :default => 131072
   # Enable SSL/TLS secured communication to Kafka broker.
-  config :ssl, :validate => :boolean, :default => false
+  config :ssl, :validate => :boolean, :default => false, :deprecated => "Use security_protocol => 'ssl'"
+  # The truststore type.
+  config :ssl_truststore_type, :validate => :string
   # The JKS truststore path to validate the Kafka broker's certificate.
   config :ssl_truststore_location, :validate => :path
   # The truststore password
   config :ssl_truststore_password, :validate => :password
+  # The keystore type.
+  config :ssl_keystore_type, :validate => :string
   # If client authentication is required, this setting stores the keystore path.
   config :ssl_keystore_location, :validate => :path
   # If client authentication is required, this setting stores the keystore password
   config :ssl_keystore_password, :validate => :password
+  # The password of the private key in the key store file.
+  config :ssl_key_password, :validate => :password
+  # Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL
+  config :security_protocol, :validate => ["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"], :default => "PLAINTEXT"
+  # http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. 
+  # This may be any mechanism for which a security provider is available.
+  # GSSAPI is the default mechanism.
+  config :sasl_mechanism, :validate => :string, :default => "GSSAPI"
+  # The Kerberos principal name that Kafka broker runs as. 
+  # This can be defined either in Kafka's JAAS config or in Kafka's config.
+  config :sasl_kerberos_service_name, :validate => :string
+  # The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization 
+  # services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client:
+  # [source,java]
+  # ----------------------------------
+  # KafkaClient {
+  #   com.sun.security.auth.module.Krb5LoginModule required
+  #   useTicketCache=true
+  #   renewTicket=true
+  #   serviceName="kafka";
+  #   };
+  # ----------------------------------
+  #
+  # Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these 
+  # to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same 
+  # `jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on 
+  # different JVM instances.
+  config :jaas_path, :validate => :path
+  # Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html
+  config :kerberos_config, :validate => :path
   # The configuration controls the maximum amount of time the server will wait for acknowledgments
   # from followers to meet the acknowledgment requirements the producer has specified with the
   # acks configuration. If the requested number of acknowledgments are not met when the timeout
@@ -143,7 +178,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
         end
         @producer.send(record)
       rescue LogStash::ShutdownSignal
-        @logger.info('Kafka producer got shutdown signal')
+        @logger.debug('Kafka producer got shutdown signal')
       rescue => e
         @logger.warn('kafka producer threw exception, restarting',
                      :exception => e)
@@ -185,19 +220,18 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
       props.put(kafka::SEND_BUFFER_CONFIG, send_buffer_bytes.to_s)
       props.put(kafka::VALUE_SERIALIZER_CLASS_CONFIG, value_serializer)
 
-      if ssl
-        if ssl_truststore_location.nil?
-          raise LogStash::ConfigurationError, "ssl_truststore_location must be set when SSL is enabled"
-        end
-        props.put("security.protocol", "SSL")
-        props.put("ssl.truststore.location", ssl_truststore_location)
-        props.put("ssl.truststore.password", ssl_truststore_password.value) unless ssl_truststore_password.nil?
+      props.put("security.protocol", security_protocol) unless security_protocol.nil?
 
-        #Client auth stuff
-        props.put("ssl.keystore.location", ssl_keystore_location) unless ssl_keystore_location.nil?
-        props.put("ssl.keystore.password", ssl_keystore_password.value) unless ssl_keystore_password.nil?
+      if security_protocol == "SSL" || ssl
+        set_trustore_keystore_config(props)
+      elsif security_protocol == "SASL_PLAINTEXT"
+        set_sasl_config(props)
+      elsif security_protocol == "SASL_SSL"
+        set_trustore_keystore_config
+        set_sasl_config
       end
 
+
       org.apache.kafka.clients.producer.KafkaProducer.new(props)
     rescue => e
       logger.error("Unable to create Kafka producer from given configuration", :kafka_error_message => e)
@@ -205,4 +239,31 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
     end
   end
 
+  def set_trustore_keystore_config(props)
+    if ssl_truststore_location.nil?
+      raise LogStash::ConfigurationError, "ssl_truststore_location must be set when SSL is enabled"
+    end
+    props.put("ssl.truststore.type", ssl_truststore_type) unless ssl_truststore_type.nil?
+    props.put("ssl.truststore.location", ssl_truststore_location)
+    props.put("ssl.truststore.password", ssl_truststore_password.value) unless ssl_truststore_password.nil?
+
+    # Client auth stuff
+    props.put("ssl.keystore.type", ssl_keystore_type) unless ssl_keystore_type.nil?
+    props.put("ssl.key.password", ssl_key_password.value) unless ssl_key_password.nil?
+    props.put("ssl.keystore.location", ssl_keystore_location) unless ssl_keystore_location.nil?
+    props.put("ssl.keystore.password", ssl_keystore_password.value) unless ssl_keystore_password.nil?
+  end
+
+  def set_sasl_config(props)
+    java.lang.System.setProperty("java.security.auth.login.config",jaas_path) unless jaas_path.nil?
+    java.lang.System.setProperty("java.security.krb5.conf",kerberos_config) unless kerberos_config.nil?
+
+    props.put("sasl.mechanism",sasl_mechanism)
+    if sasl_mechanism == "GSSAPI" && sasl_kerberos_service_name.nil?
+      raise LogStash::ConfigurationError, "sasl_kerberos_service_name must be specified when SASL mechanism is GSSAPI"
+    end
+
+    props.put("sasl.kerberos.service.name",sasl_kerberos_service_name)
+  end
+
 end #class LogStash::Outputs::Kafka

data/logstash-output-kafka.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-kafka'
-  s.version         = '5.0.4'
+  s.version         = '5.1.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = 'Output events to a Kafka topic. This uses the Kafka Producer API to write messages to a topic on the broker'
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-kafka
 version: !ruby/object:Gem::Version
-  version: 5.0.4
+  version: 5.1.0
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-08-19 00:00:00.000000000 Z
+date: 2016-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement