fluent-plugin-kafka 0.6.6 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f03db16ea9d9f3e937b8f2e5ee3840ac92929f98
-  data.tar.gz: 8f5db823ac54b6154c38571585e18915b38d0e71
+  metadata.gz: 82c28fd4a4b133b7ad2685d4bd65b49ce4640b51
+  data.tar.gz: 9e215032a9f21a622abd34185e0d4137daa69fd6
 SHA512:
-  metadata.gz: 5a032b3b1cbabc0ec9678257151cba70d98338999bd028c865419790817259e63eb7172c24a9956db3bfe0ee2b35a6c27615bbb1e6c39e466dfac39cf9282bb0
-  data.tar.gz: fa7da134390f0532f399b5e0d802c377e21d917d4ed606e0ec6d1df1b0a1e81c54e37b7897643fc191cf3d41ba420e1859424170b3fbe07fd7039413300c61c5
+  metadata.gz: 2cabd4dd25b39d84cd1785d4f28b0733719504ceef7c8e39fe55c4179232869de53db740007f423c0825a316a324a52d9bf806795fc3581b6e0f9b59bbcd133b
+  data.tar.gz: 4ccf05ed9a4776a6380988b8ab9c603205cb111c39715c591baaf9473103d882e7535b4369a8189ad33779d6543bc6c01b97b5ac3682d38976ad9fbe179af4b2

data/ChangeLog

@@ -1,3 +1,9 @@
+Release 0.7.0 - 2018/02/27
+
+	* output: SASL PLAIN/SCRAM support
+	* input: Fix TimeParser location bug in v1
+	* out_kafka2: Fix warning condition for tag chunk key
+
 Release 0.6.6 - 2017/12/25
 
 	* output: Make topic/partition/partition_key/message_key configurable

data/README.md

@@ -1,6 +1,6 @@
 # fluent-plugin-kafka, a plugin for [Fluentd](http://fluentd.org)
 
-[![Build Status](https://travis-ci.org/htgc/fluent-plugin-kafka.svg?branch=master)](https://travis-ci.org/htgc/fluent-plugin-kafka)
+[![Build Status](https://travis-ci.org/fluent/fluent-plugin-kafka.svg?branch=master)](https://travis-ci.org/fluent/fluent-plugin-kafka)
 
 A fluentd plugin to both consume and produce data for Apache Kafka.
 
@@ -213,7 +213,7 @@ If key name `message_key` exists in a message, this plugin publishes the value o
 
 ### Output plugin
 
-This plugin is for v1.0. This will be `out_kafka` plugin in the feature.
+This plugin is for v1.0. This will be `out_kafka` plugin in the future.
 
     <match app.**>
       @type kafka2
@@ -242,7 +242,7 @@ This plugin is for v1.0. This will be `out_kafka` plugin in the feature.
       # See fluentd document for buffer related parameters: http://docs.fluentd.org/articles/buffer-plugin-overview
       # Buffer chunk key should be same with topic_key. If value is not found in the record, default_topic is used.
       <buffer topic>
-        flush_interavl 10s
+        flush_interval 10s
       </buffer>
 
       # ruby-kafka producer options

data/fluent-plugin-kafka.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.6.6'
+  gem.version       = '0.7.0'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/in_kafka.rb

@@ -108,7 +108,11 @@ class Fluent::KafkaInput < Fluent::Input
     @parser_proc = setup_parser
 
     if @use_record_time and @time_format
-      @time_parser = Fluent::TextParser::TimeParser.new(@time_format)
+      if defined?(Fluent::TimeParser)
+        @time_parser = Fluent::TimeParser.new(@time_format)
+      else
+        @time_parser = Fluent::TextParser::TimeParser.new(@time_format)
+      end
     end
   end
 

data/lib/fluent/plugin/in_kafka_group.rb

@@ -108,7 +108,11 @@ class Fluent::KafkaGroupInput < Fluent::Input
     @fetch_opts[:min_bytes] = @min_bytes if @min_bytes
 
     if @use_record_time and @time_format
-      @time_parser = Fluent::TextParser::TimeParser.new(@time_format)
+      if defined?(Fluent::TimeParser)
+        @time_parser = Fluent::TimeParser.new(@time_format)
+      else
+        @time_parser = Fluent::TextParser::TimeParser.new(@time_format)
+      end
     end
   end
 

data/lib/fluent/plugin/kafka_plugin_util.rb

@@ -31,6 +31,12 @@ module Fluent
                        :desc => "a Kerberos principal to use with SASL authentication (GSSAPI)."
           config_param :keytab, :string, :default => nil,
                        :desc => "a filepath to Kerberos keytab. Must be used with principal."
+          config_param :username, :string, :default => nil,
+                       :desc => "a username when using PLAIN/SCRAM SASL authentication"
+          config_param :password, :string, :default => nil,
+                       :desc => "a password when using PLAIN/SCRAM SASL authentication"
+          config_param :scram_mechanism, :string, :default => nil,
+                       :desc => "if set, use SCRAM authentication with specified mechanism. When unset, default to PLAIN authentication"
         }
       end
     end

data/lib/fluent/plugin/out_kafka.rb

@@ -97,9 +97,19 @@ DESC
     end
     begin
       if @seed_brokers.length > 0
-        @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
-                           ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
-                           sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab)
+        if @scram_mechanism != nil && @username != nil && @password != nil
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                             sasl_scram_username: @username, sasl_scram_password: @password, sasl_scram_mechanism: @scram_mechanism)
+        elsif @username != nil && @password != nil
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                             sasl_plain_username: @username, sasl_plain_password: @password)
+        else
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                             sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab)
+        end
         log.info "initialized kafka producer: #{@client_id}"
       else
         log.warn "No brokers found on Zookeeper"
@@ -115,7 +125,7 @@ DESC
     if @zookeeper
       require 'zookeeper'
     else
-      @seed_brokers = @brokers.match(",").nil? ? [@brokers] : @brokers.split(",")
+      @seed_brokers = @brokers.split(",")
       log.info "brokers has been set directly: #{@seed_brokers}"
     end
 

data/lib/fluent/plugin/out_kafka2.rb

@@ -74,9 +74,19 @@ DESC
     def refresh_client(raise_error = true)
       begin
         logger = @get_kafka_client_log ? log : nil
-        @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
-                           ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
-                           sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab)
+        if @scram_mechanism != nil && @username != nil && @password != nil
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                             sasl_scram_username: @username, sasl_scram_password: @password, sasl_scram_mechanism: @scram_mechanism)
+        elsif @username != nil && @password != nil
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                             sasl_plain_username: @username, sasl_plain_password: @password)
+        else
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                             sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab)
+        end
         log.info "initialized kafka producer: #{@client_id}"
       rescue Exception => e
         if raise_error # During startup, error should be reported to engine and stop its phase for safety.
@@ -106,11 +116,11 @@ DESC
       @formatter_proc = setup_formatter(formatter_conf)
 
       if @default_topic.nil?
-        if @chunk_keys.include?('topic') && !@chunk_keys.include?('tag')
+        if @chunk_keys.include?('topic') && !@chunk_key_tag
           log.warn "Use 'topic' field of event record for topic but no fallback. Recommend to set default_topic or set 'tag' in buffer chunk keys like <buffer topic,tag>"
         end
       else
-        if @chunk_keys.include?('tag')
+        if @chunk_key_tag
           log.warn "default_topic is set. Fluentd's event tag is not used for topic"
         end
       end

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -119,9 +119,19 @@ DESC
     begin
       if @seed_brokers.length > 0
         logger = @get_kafka_client_log ? log : nil
-        @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
-                           ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
-                           sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab)
+        if @scram_mechanism != nil && @username != nil && @password != nil
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                             sasl_scram_username: @username, sasl_scram_password: @password, sasl_scram_mechanism: @scram_mechanism)
+        elsif @username != nil && @password != nil
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                             sasl_plain_username: @username, sasl_plain_password: @password)
+        else
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                             sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab)
+        end
         log.info "initialized kafka producer: #{@client_id}"
       else
         log.warn "No brokers found on Zookeeper"
@@ -141,7 +151,7 @@ DESC
     if @zookeeper
       require 'zookeeper'
     else
-      @seed_brokers = @brokers.match(",").nil? ? [@brokers] : @brokers.split(",")
+      @seed_brokers = @brokers.split(",")
       log.info "brokers has been set directly: #{@seed_brokers}"
     end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.6.6
+  version: 0.7.0
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-12-25 00:00:00.000000000 Z
+date: 2018-02-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd