fluent-plugin-kafka 0.7.6 → 0.7.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1f69043ece5ca612bdca6988b3fea137e958832b
-  data.tar.gz: 1b784d0fca451efd86fa532f868033f5605b21e7
+  metadata.gz: 44307aa17ae864f8e892b8c0eeb95066fe50c89c
+  data.tar.gz: 13f83473d1835e7613ec989f21e017f99236c0e3
 SHA512:
-  metadata.gz: c081880f8953d5e97c7ff31f90a84ead32ea538103d215f8d3cb65287a87f39e3acf03be0a5f2af15ce4ff9a2816c114bf5b2d715afde7e90ebd6663cc237c0c
-  data.tar.gz: e1d65842eaadb2edc711be10612cc8a5d69f648e0934e49c5544360aaca798842cbac021cd21080fcdc4700c943773a571aedc2083f23df8d32fed3959f7c95a
+  metadata.gz: f62828d69cc47006dfc5fba8ff31ecc6db33c867144c9a348ff952288aed48d3c5e8ebb7c3b625369cb694b3c0f5d98bb80df193e6545de4d8d6db6338c58539
+  data.tar.gz: f2a2d16d81a2eeabbfe597d15cd2124e7a0f1b2e625f136d5b45fae7ff0ba94563ade1383dd9638c23be79ee8326bef9520ce135be56553f95010784fc9045ed

data/ChangeLog

@@ -1,3 +1,10 @@
+Release 0.7.7 - 2018/08/27
+
+	* in_kafka_group: Add retry_limit and disable_retry_limit parameters
+	* input: Support SSL parameters
+	* Add ssl_ca_certs_from_system parameter
+	* Make password parameter secret
+
 Release 0.7.6 - 2018/08/15
 
 	* out_kafka2: Add ignore_exceptions and exception_backup parameters

data/README.md

@@ -37,6 +37,7 @@ If you want to use zookeeper related parameters, you also need to install zookee
 - ssl_ca_cert
 - ssl_client_cert
 - ssl_client_cert_key
+- ssl_ca_certs_from_system
 
 Set path to SSL related files. See [Encryption and Authentication using SSL](https://github.com/zendesk/ruby-kafka#encryption-and-authentication-using-ssl) for more detail.
 
@@ -280,10 +281,10 @@ This plugin uses ruby-kafka producer for writing data. For performance and relia
       # ruby-kafka producer options
       max_send_retries    (integer)     :default => 1
       required_acks       (integer)     :default => -1
-      ack_timeout         (integer)     :default => nil (Use default of ruby-kafka) 
+      ack_timeout         (integer)     :default => nil (Use default of ruby-kafka)
       compression_codec   (gzip|snappy) :default => nil
       max_buffer_size     (integer)     :default => nil (Use default of ruby-kafka)
-      max_buffer_bytesize (integer)     :default => nil (Use default of ruby-kafka) 
+      max_buffer_bytesize (integer)     :default => nil (Use default of ruby-kafka)
     </match>
 
 This plugin also supports ruby-kafka related parameters. See Buffered output plugin section.

data/fluent-plugin-kafka.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.7.6'
+  gem.version       = '0.7.7'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/in_kafka.rb

@@ -173,11 +173,21 @@ class Fluent::KafkaInput < Fluent::Input
     opt[:max_wait_time] = @max_wait_time if @max_wait_time
     opt[:min_bytes] = @min_bytes if @min_bytes
 
-    @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id,
-                       ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
-                       ssl_client_cert: read_ssl_file(@ssl_client_cert),
-                       ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
-                       sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab)
+    if @scram_mechanism != nil && @username != nil && @password != nil
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+                         ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                         ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_scram_username: @username, sasl_scram_password: @password,
+                         sasl_scram_mechanism: @scram_mechanism)
+    elsif @username != nil && @password != nil
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+                         ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                         ssl_ca_certs_from_system: @ssl_ca_certs_from_system,sasl_plain_username: @username, sasl_plain_password: @password)
+    else
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+                         ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                         ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab)
+    end
+
     @zookeeper = Zookeeper.new(@offset_zookeeper) if @offset_zookeeper
 
     @topic_watchers = @topic_list.map {|topic_entry|

data/lib/fluent/plugin/in_kafka_group.rb

@@ -11,7 +11,7 @@ class Fluent::KafkaGroupInput < Fluent::Input
                :desc => "Consumer group name, must set."
   config_param :topics, :string,
                :desc => "Listening topics(separate with comma',')."
-  config_param :client_id, :string, :default => 'kafka'               
+  config_param :client_id, :string, :default => 'kafka'
   config_param :format, :string, :default => 'json',
                :desc => "Supported format: (json|text|ltsv|msgpack)"
   config_param :message_key, :string, :default => 'message',
@@ -30,6 +30,10 @@ class Fluent::KafkaGroupInput < Fluent::Input
                :desc => "Set kafka's message key to this field"
 
   config_param :retry_wait_seconds, :integer, :default => 30
+  config_param :disable_retry_limit, :bool, :default => false,
+               :desc => "If set true, it disables retry_limit and make Fluentd retry indefinitely (default: false)"
+  config_param :retry_limit, :integer, :default => 10,
+               :desc => "The maximum number of retries for connecting kafka (default: 10)"
   # Kafka consumer options
   config_param :max_bytes, :integer, :default => 1048576,
                :desc => "Maximum number of bytes to fetch."
@@ -67,6 +71,7 @@ class Fluent::KafkaGroupInput < Fluent::Input
     require 'kafka'
 
     @time_parser = nil
+    @retry_count = 1
   end
 
   def _config_to_array(config)
@@ -141,11 +146,21 @@ class Fluent::KafkaGroupInput < Fluent::Input
   def start
     super
 
-    @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id,
-                       ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
-                       ssl_client_cert: read_ssl_file(@ssl_client_cert),
-                       ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
-                       sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab)
+    if @scram_mechanism != nil && @username != nil && @password != nil
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+                         ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                         ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_scram_username: @username, sasl_scram_password: @password,
+                         sasl_scram_mechanism: @scram_mechanism)
+    elsif @username != nil && @password != nil
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+                         ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                         ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_plain_username: @username, sasl_plain_password: @password)
+    else
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+                         ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                         ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab)
+    end
+
     @consumer = setup_consumer
     @thread = Thread.new(&method(:run))
   end
@@ -170,21 +185,28 @@ class Fluent::KafkaGroupInput < Fluent::Input
     }
     consumer
   end
-  
+
   def reconnect_consumer
     log.warn "Stopping Consumer"
     consumer = @consumer
     @consumer = nil
-    consumer.stop
-    log.warn "Could not connect to broker. Next retry will be in #{@retry_wait_seconds} seconds"
+    if consumer 
+      consumer.stop
+    end
+    log.warn "Could not connect to broker. retry_time:#{@retry_count}. Next retry will be in #{@retry_wait_seconds} seconds"
+    @retry_count = @retry_count + 1
     sleep @retry_wait_seconds
     @consumer = setup_consumer
     log.warn "Re-starting consumer #{Time.now.to_s}"
+    @retry_count = 0 
   rescue =>e
     log.error "unexpected error during re-starting consumer object access", :error => e.to_s
     log.error_backtrace
+    if @retry_count <= @retry_limit or disable_retry_limit
+      reconnect_consumer
+    end
   end
-  
+
   def run
     while @consumer
       begin
@@ -254,3 +276,4 @@ class Fluent::KafkaGroupInput < Fluent::Input
     end
   end
 end
+

data/lib/fluent/plugin/kafka_plugin_util.rb

@@ -10,6 +10,8 @@ module Fluent
                        :desc => "a PEM encoded client cert to use with and SSL connection. Must be used in combination with ssl_client_cert_key."
           config_param :ssl_client_cert_key, :string, :default => nil,
                        :desc => "a PEM encoded client cert key to use with and SSL connection. Must be used in combination with ssl_client_cert."
+          config_param :ssl_ca_certs_from_system, :bool, :default => false,
+                       :desc => "this configures the store to look up CA certificates from the system default certificate store on an as needed basis. The location of the store can usually be determined by: OpenSSL::X509::DEFAULT_CERT_FILE."
         }
       end
 
@@ -39,7 +41,7 @@ module Fluent
                        :desc => "a filepath to Kerberos keytab. Must be used with principal."
           config_param :username, :string, :default => nil,
                        :desc => "a username when using PLAIN/SCRAM SASL authentication"
-          config_param :password, :string, :default => nil,
+          config_param :password, :string, :default => nil, secret: true,
                        :desc => "a password when using PLAIN/SCRAM SASL authentication"
           config_param :scram_mechanism, :string, :default => nil,
                        :desc => "if set, use SCRAM authentication with specified mechanism. When unset, default to PLAIN authentication"

data/lib/fluent/plugin/out_kafka.rb

@@ -103,15 +103,15 @@ DESC
       if @seed_brokers.length > 0
         if @scram_mechanism != nil && @username != nil && @password != nil
           @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
-                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
                              sasl_scram_username: @username, sasl_scram_password: @password, sasl_scram_mechanism: @scram_mechanism)
         elsif @username != nil && @password != nil
           @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
-                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
                              sasl_plain_username: @username, sasl_plain_password: @password)
         else
           @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
-                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
                              sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab)
         end
         log.info "initialized kafka producer: #{@client_id}"

data/lib/fluent/plugin/out_kafka2.rb

@@ -79,15 +79,15 @@ DESC
         logger = @get_kafka_client_log ? log : nil
         if @scram_mechanism != nil && @username != nil && @password != nil
           @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
-                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
                              sasl_scram_username: @username, sasl_scram_password: @password, sasl_scram_mechanism: @scram_mechanism)
         elsif @username != nil && @password != nil
           @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
-                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
                              sasl_plain_username: @username, sasl_plain_password: @password)
         else
           @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
-                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
                              sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab)
         end
         log.info "initialized kafka producer: #{@client_id}"

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -125,15 +125,15 @@ DESC
         logger = @get_kafka_client_log ? log : nil
         if @scram_mechanism != nil && @username != nil && @password != nil
           @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
-                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
                              sasl_scram_username: @username, sasl_scram_password: @password, sasl_scram_mechanism: @scram_mechanism)
         elsif @username != nil && @password != nil
           @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
-                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
                              sasl_plain_username: @username, sasl_plain_password: @password)
         else
           @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
-                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
                              sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab)
         end
         log.info "initialized kafka producer: #{@client_id}"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.7.6
+  version: 0.7.7
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-16 00:00:00.000000000 Z
+date: 2018-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd