fluent-plugin-kafka 0.5.7 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cb704a5814ca89bb71f2a9ead76631881a0c3f9e
-  data.tar.gz: 2b6eb6c80c5060a650f17e97bfbee67b478d0e79
+  metadata.gz: 724da4a2036ca9c067bb14f5dce7a4011120b486
+  data.tar.gz: dcf10e1242df2a44b7cab4ac3ca48df545131bdb
 SHA512:
-  metadata.gz: b2e958ccbc9af8a9b0e4cffc87ad54054d5a579f584ec820a09eb02bcf9392c394938b3c7ad276a2a15b27f8d3d137c00b5f089566b2a03c5bcd0939528b6a68
-  data.tar.gz: 96ac9c4cf08269eef363677dc7dd38a316b54c82b5ca1504661770e210c7dd2c52bb1d7e1edbbeaa3a156b78e332bbe3170bc271ce200b2a46fa964beb59477b
+  metadata.gz: 5fa1fdc5af0678ea8212488021b261c5c1f06b129c6a88b796c24cc6dcc306647f770cae276003249e0f1b36a27a5f0d95e085cf2af2e6af0ebf7fad1a6a4257
+  data.tar.gz: e304d72397ffb647e511d62e53ce467083f583f67e020ce376fc4d1270a957f47908b0204a27184a12b4c9f7b96a2ae724ca7e15872235663ef6c840d5de4e67

data/ChangeLog

@@ -1,3 +1,7 @@
+Release 0.6.0 - 2017/07/25
+
+	* Add principal and keytab parameters for SASL support
+
 Release 0.5.7 - 2017/07/13
 
 	* out_kafka_buffered: Add kafka_agg_max_messages parameter

data/README.md

@@ -32,12 +32,21 @@ If you want to use zookeeper related parameters, you also need to install zookee
 
 ### Common parameters
 
+#### SSL authentication
+
 - ssl_ca_cert
 - ssl_client_cert
 - ssl_client_cert_key
 
 Set path to SSL related files. See [Encryption and Authentication using SSL](https://github.com/zendesk/ruby-kafka#encryption-and-authentication-using-ssl) for more detail.
 
+#### SASL authentication
+
+- principal
+- keytab
+
+Set principal and path to keytab for SASL/GSSAPI authentication. See [Authentication using SASL](https://github.com/zendesk/ruby-kafka#authentication-using-sasl) for more details.
+
 ### Input plugin (@type 'kafka')
 
 Consume events by single consumer.
@@ -150,7 +159,7 @@ This plugin uses ruby-kafka producer for writing data. This plugin works with re
 `<formatter name>` of `output_data_type` uses fluentd's formatter plugins. See [formatter article](http://docs.fluentd.org/articles/formatter-plugin-overview).
 
 ruby-kafka sometimes returns `Kafka::DeliveryFailed` error without good information.
-In this case, `get_kafka_client_log` is useful for identifying the error cause.    
+In this case, `get_kafka_client_log` is useful for identifying the error cause.
 ruby-kafka's log is routed to fluentd log so you can see ruby-kafka's log in fluentd logs.
 
 Supports following ruby-kafka's producer options.

data/fluent-plugin-kafka.gemspec

@@ -12,12 +12,12 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.5.7'
+  gem.version       = '0.6.0'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
   gem.add_dependency 'ltsv'
-  gem.add_dependency 'ruby-kafka', '~> 0.3.16'
+  gem.add_dependency 'ruby-kafka', '~> 0.4.0'
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "test-unit", ">= 3.0.8"
 end

data/lib/fluent/plugin/in_kafka.rb

@@ -46,6 +46,7 @@ class Fluent::KafkaInput < Fluent::Input
                :desc => "Smallest amount of data the server should send us."
 
   include Fluent::KafkaPluginUtil::SSLSettings
+  include Fluent::KafkaPluginUtil::SaslSettings
 
   unless method_defined?(:router)
     define_method("router") { Fluent::Engine }
@@ -159,7 +160,8 @@ class Fluent::KafkaInput < Fluent::Input
     @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id,
                        ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                        ssl_client_cert: read_ssl_file(@ssl_client_cert),
-                       ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key))
+                       ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                       sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab)
     @zookeeper = Zookeeper.new(@offset_zookeeper) if @offset_zookeeper
 
     @topic_watchers = @topic_list.map {|topic_entry|

data/lib/fluent/plugin/in_kafka_group.rb

@@ -43,6 +43,7 @@ class Fluent::KafkaGroupInput < Fluent::Input
                :desc => "Whether to start from the beginning of the topic or just subscribe to new messages being produced"
 
   include Fluent::KafkaPluginUtil::SSLSettings
+  include Fluent::KafkaPluginUtil::SaslSettings
 
   class ForShutdown < StandardError
   end
@@ -125,7 +126,8 @@ class Fluent::KafkaGroupInput < Fluent::Input
     @kafka = Kafka.new(seed_brokers: @brokers,
                        ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                        ssl_client_cert: read_ssl_file(@ssl_client_cert),
-                       ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key))
+                       ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                       sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab)
     @consumer = setup_consumer
     @thread = Thread.new(&method(:run))
   end

data/lib/fluent/plugin/kafka_plugin_util.rb

@@ -18,5 +18,16 @@ module Fluent
         File.read(path)
       end
     end
+
+    module SaslSettings
+      def self.included(klass)
+        klass.instance_eval {
+          config_param :principal, :string, :default => nil,
+                       :desc => "a Kerberos principal to use with SASL authentication (GSSAPI)."
+          config_param :keytab, :string, :default => nil,
+                       :desc => "a filepath to Kerberos keytab. Must be used with principal."
+        }
+      end
+    end
   end
 end

data/lib/fluent/plugin/out_kafka.rb

@@ -67,6 +67,7 @@ requires activesupport gem - records will be generated under fluent_kafka_stats.
 DESC
 
   include Fluent::KafkaPluginUtil::SSLSettings
+  include Fluent::KafkaPluginUtil::SaslSettings
 
   attr_accessor :output_data_type
   attr_accessor :field_separator
@@ -97,7 +98,8 @@ DESC
     begin
       if @seed_brokers.length > 0
         @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
-                           ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key))
+                           ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                           sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab)
         log.info "initialized kafka producer: #{@client_id}"
       else
         log.warn "No brokers found on Zookeeper"

data/lib/fluent/plugin/out_kafka2.rb

@@ -59,6 +59,7 @@ DESC
     end
 
     include Fluent::KafkaPluginUtil::SSLSettings
+    include Fluent::KafkaPluginUtil::SaslSettings
 
     def initialize
       super
@@ -70,7 +71,8 @@ DESC
       begin
         logger = @get_kafka_client_log ? log : nil
         @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
-                           ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key))
+                           ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                           sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab)
         log.info "initialized kafka producer: #{@client_id}"
       rescue Exception => e
         if raise_error # During startup, error should be reported to engine and stop its phase for safety.

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -75,6 +75,7 @@ requires activesupport gem - records will be generated under fluent_kafka_stats.
 DESC
 
   include Fluent::KafkaPluginUtil::SSLSettings
+  include Fluent::KafkaPluginUtil::SaslSettings
 
   attr_accessor :output_data_type
   attr_accessor :field_separator
@@ -109,7 +110,8 @@ DESC
       if @seed_brokers.length > 0
         logger = @get_kafka_client_log ? log : nil
         @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
-                           ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key))
+                           ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                           sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab)
         log.info "initialized kafka producer: #{@client_id}"
       else
         log.warn "No brokers found on Zookeeper"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.5.7
+  version: 0.6.0
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-12 00:00:00.000000000 Z
+date: 2017-07-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -51,14 +51,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.16
+        version: 0.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.16
+        version: 0.4.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement