logstash-input-kafka 5.0.6 → 5.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e514f65b45c9b98ff3e293b2e7a48e683115d18d
-  data.tar.gz: 5aca38ab5940c52a141addab0504652262dd727b
+  metadata.gz: a9c0d93413c3ea01f0181364f4e2bb5eb93c8b56
+  data.tar.gz: 97e2e2659773e8c46381388d206823b694c431e0
 SHA512:
-  metadata.gz: b5f313584a6b291bf1d409c8a4a8ed242d631468b0252809f9aa3d10b3eec0527723d2c995c51208db050dabfe1dc342cf563dab4b56b849c39c9080ae4ff24f
-  data.tar.gz: 70546e9093fbe5412ba7e258b0c6de02d678ffc5649a6a4c6d17f1b9eaf35325311fef03c84de3eee5b171a631d5f4dcfc940df582276a35238cabe06f52f821
+  metadata.gz: 99fdcf04705b3d6376bea25aa8cbefbd303e8c7d4d6107b754ba99f3e4ad40010919f576bfa64154a0c64a03e438bc24bac40a1889a044d193b674a9a6f8715a
+  data.tar.gz: eadd2571707d7eeaacef2021d1b2698ba752a2e4bbc38471ef281279f3c9a07650d27291a34fe5791ac646f372658723e4a8d9a0f555cfe6e52c44807f494bca

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 5.1.0
+  - Add Kerberos authentication support.
+
 ## 5.0.6
   - default `poll_timeout_ms` to 100ms
 

data/lib/logstash/inputs/kafka.rb

@@ -145,15 +145,49 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
   # Time kafka consumer will wait to receive new messages from topics
   config :poll_timeout_ms, :validate => :number, :default => 100
   # Enable SSL/TLS secured communication to Kafka broker.
-  config :ssl, :validate => :boolean, :default => false
+  config :ssl, :validate => :boolean, :default => false, :deprecated => "Use security_protocol => 'ssl'"
+  # The truststore type.
+  config :ssl_truststore_type, :validate => :string
   # The JKS truststore path to validate the Kafka broker's certificate.
   config :ssl_truststore_location, :validate => :path
   # The truststore password
   config :ssl_truststore_password, :validate => :password
+  # The keystore type.
+  config :ssl_keystore_type, :validate => :string
   # If client authentication is required, this setting stores the keystore path.
   config :ssl_keystore_location, :validate => :path
   # If client authentication is required, this setting stores the keystore password
   config :ssl_keystore_password, :validate => :password
+  # The password of the private key in the key store file.
+  config :ssl_key_password, :validate => :password
+  # Security protocol to use, which can be either of PLAINTEXT,SSL,SASL_PLAINTEXT,SASL_SSL
+  config :security_protocol, :validate => ["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"], :default => "PLAINTEXT"
+  # http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. 
+  # This may be any mechanism for which a security provider is available.
+  # GSSAPI is the default mechanism.
+  config :sasl_mechanism, :validate => :string, :default => "GSSAPI"
+  # The Kerberos principal name that Kafka broker runs as. 
+  # This can be defined either in Kafka's JAAS config or in Kafka's config.
+  config :sasl_kerberos_service_name, :validate => :string
+  # The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization 
+  # services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client:
+  # [source,java]
+  # ----------------------------------
+  # KafkaClient {
+  #   com.sun.security.auth.module.Krb5LoginModule required
+  #   useTicketCache=true
+  #   renewTicket=true
+  #   serviceName="kafka";
+  #   };
+  # ----------------------------------
+  #
+  # Please note that specifying `jaas_path` and `kerberos_config` in the config file will add these  
+  # to the global JVM system properties. This means if you have multiple Kafka inputs, all of them would be sharing the same 
+  # `jaas_path` and `kerberos_config`. If this is not desirable, you would have to run separate instances of Logstash on 
+  # different JVM instances.
+  config :jaas_path, :validate => :path
+  # Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html
+  config :kerberos_config, :validate => :path
   # Option to add Kafka metadata like topic, message size to the event.
   # This will add a field named `kafka` to the logstash event containing the following attributes:
   #   `topic`: The topic this message is associated with
@@ -252,14 +286,15 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
       props.put(kafka::SESSION_TIMEOUT_MS_CONFIG, session_timeout_ms) unless session_timeout_ms.nil?
       props.put(kafka::VALUE_DESERIALIZER_CLASS_CONFIG, value_deserializer_class)
 
-      if ssl
-        props.put("security.protocol", "SSL")
-        props.put("ssl.truststore.location", ssl_truststore_location)
-        props.put("ssl.truststore.password", ssl_truststore_password.value) unless ssl_truststore_password.nil?
+      props.put("security.protocol", security_protocol) unless security_protocol.nil?
 
-        #Client auth stuff
-        props.put("ssl.keystore.location", ssl_keystore_location) unless ssl_keystore_location.nil?
-        props.put("ssl.keystore.password", ssl_keystore_password.value) unless ssl_keystore_password.nil?
+      if security_protocol == "SSL"
+        set_trustore_keystore_config(props)
+      elsif security_protocol == "SASL_PLAINTEXT"
+        set_sasl_config(props)
+      elsif security_protocol == "SASL_SSL"
+        set_trustore_keystore_config
+        set_sasl_config
       end
 
       org.apache.kafka.clients.consumer.KafkaConsumer.new(props)
@@ -268,4 +303,28 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base
       throw e
     end
   end
+
+  def set_trustore_keystore_config(props)
+    props.put("ssl.truststore.type", ssl_truststore_type) unless ssl_truststore_type.nil?
+    props.put("ssl.truststore.location", ssl_truststore_location)
+    props.put("ssl.truststore.password", ssl_truststore_password.value) unless ssl_truststore_password.nil?
+
+    # Client auth stuff
+    props.put("ssl.keystore.type", ssl_keystore_type) unless ssl_keystore_type.nil?
+    props.put("ssl.key.password", ssl_key_password.value) unless ssl_key_password.nil?
+    props.put("ssl.keystore.location", ssl_keystore_location) unless ssl_keystore_location.nil?
+    props.put("ssl.keystore.password", ssl_keystore_password.value) unless ssl_keystore_password.nil?
+  end
+
+  def set_sasl_config(props)
+    java.lang.System.setProperty("java.security.auth.login.config",jaas_path) unless jaas_path.nil?
+    java.lang.System.setProperty("java.security.krb5.conf",kerberos_config) unless kerberos_config.nil?
+
+    props.put("sasl.mechanism",sasl_mechanism)
+    if sasl_mechanism == "GSSAPI" && sasl_kerberos_service_name.nil?
+      raise LogStash::ConfigurationError, "sasl_kerberos_service_name must be specified when SASL mechanism is GSSAPI"
+    end
+
+    props.put("sasl.kerberos.service.name",sasl_kerberos_service_name)
+  end
 end #class LogStash::Inputs::Kafka

data/logstash-input-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-kafka'
-  s.version         = '5.0.6'
+  s.version         = '5.1.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = 'This input will read events from a Kafka topic. It uses the high level consumer API provided by Kafka to read messages from the broker'
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-kafka
 version: !ruby/object:Gem::Version
-  version: 5.0.6
+  version: 5.1.0
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-11-08 00:00:00.000000000 Z
+date: 2016-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement