fluent-plugin-kafka 0.6.0 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 724da4a2036ca9c067bb14f5dce7a4011120b486
-  data.tar.gz: dcf10e1242df2a44b7cab4ac3ca48df545131bdb
+  metadata.gz: c15b97c9566c5fcae1478eaeef953786e4f58412
+  data.tar.gz: 0a12eca981a1a4e44227b33142dac987a8e36c96
 SHA512:
-  metadata.gz: 5fa1fdc5af0678ea8212488021b261c5c1f06b129c6a88b796c24cc6dcc306647f770cae276003249e0f1b36a27a5f0d95e085cf2af2e6af0ebf7fad1a6a4257
-  data.tar.gz: e304d72397ffb647e511d62e53ce467083f583f67e020ce376fc4d1270a957f47908b0204a27184a12b4c9f7b96a2ae724ca7e15872235663ef6c840d5de4e67
+  metadata.gz: c190e02a5a0aafd3784b0eab74e1adbc1a291347e99d5154446253f7116a22b4c1ba969bea27da76ed918a156417b3150eb1fc45b73c7cf2a3e777cc0ae0c4b2
+  data.tar.gz: 13f146fffce6d05a6519c0e7dd8e04c8dde01d85345f9dcb3c3ab8de4792d6666d8deb704422fc8cea93c95786308ef773decfb31d2704e8b61542ffa0ef2454

data/ChangeLog

@@ -1,3 +1,10 @@
+Release 0.6.1 - 2017/08/30
+
+	* Add stats and datadog monitoring support
+	* ssl_ca_certs now accepts multiple paths
+	* Fix bug by ruby-kafka 0.4.1 changes
+	* Update ruby-kafka dependency to v0.4.1
+
 Release 0.6.0 - 2017/07/25
 
 	* Add principal and keytab parameters for SASL support

data/README.md

@@ -153,7 +153,8 @@ This plugin uses ruby-kafka producer for writing data. This plugin works with re
       kafka_agg_max_bytes          (integer)     :default => 4096
       kafka_agg_max_messages       (integer)     :default => nil (No limit)
       max_send_limit_bytes         (integer)     :default => nil (No drop)
-      discard_kafka_delivery_failed   (bool)        :default => false (No discard)
+      discard_kafka_delivery_failed   (bool)     :default => false (No discard)
+      monitoring_list              (array)       :default => []
     </match>
 
 `<formatter name>` of `output_data_type` uses fluentd's formatter plugins. See [formatter article](http://docs.fluentd.org/articles/formatter-plugin-overview).
@@ -172,6 +173,9 @@ Supports following ruby-kafka's producer options.
 - kafka_agg_max_messages - default: nil - Maximum number of messages to include in one batch transmission.
 - max_send_limit_bytes - default: nil - Max byte size to send message to avoid MessageSizeTooLarge. For example, if you set 1000000(message.max.bytes in kafka), Message more than 1000000 byes will be dropped.
 - discard_kafka_delivery_failed - default: false - discard the record where [Kafka::DeliveryFailed](http://www.rubydoc.info/gems/ruby-kafka/Kafka/DeliveryFailed) occurred
+- monitoring_list - default: [] - library to be used to monitor. statsd and datadog are supported
+
+If you want to know about detail of monitoring, see also https://github.com/zendesk/ruby-kafka#monitoring
 
 See also [Kafka::Client](http://www.rubydoc.info/gems/ruby-kafka/Kafka/Client) for more detailed documentation about ruby-kafka.
 

data/fluent-plugin-kafka.gemspec

@@ -12,12 +12,12 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.6.0'
+  gem.version       = '0.6.1'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
   gem.add_dependency 'ltsv'
-  gem.add_dependency 'ruby-kafka', '~> 0.4.0'
+  gem.add_dependency 'ruby-kafka', '~> 0.4.1'
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "test-unit", ">= 3.0.8"
 end

data/lib/fluent/plugin/kafka_plugin_util.rb

@@ -4,7 +4,7 @@ module Fluent
       def self.included(klass)
         klass.instance_eval {
           # https://github.com/zendesk/ruby-kafka#encryption-and-authentication-using-ssl
-          config_param :ssl_ca_cert, :string, :default => nil,
+          config_param :ssl_ca_cert, :array, :value_type => :string, :default => nil,
                        :desc => "a PEM encoded CA cert to use with and SSL connection."
           config_param :ssl_client_cert, :string, :default => nil,
                        :desc => "a PEM encoded client cert to use with and SSL connection. Must be used in combination with ssl_client_cert_key."
@@ -15,7 +15,12 @@ module Fluent
 
       def read_ssl_file(path)
         return nil if path.nil?
-        File.read(path)
+
+        if path.is_a?(Array)
+          path.map { |fp| File.read(fp) }
+        else
+          File.read(path)
+        end
       end
     end
 

data/lib/fluent/plugin/kafka_producer_ext.rb

@@ -19,8 +19,7 @@ module Kafka
         topic,
         partition,
         partition_key,
-        create_time,
-        key.to_s.bytesize + value.to_s.bytesize
+        create_time
       )
 
       @target_topics.add(topic)
@@ -88,8 +87,7 @@ module Kafka
         @topic,
         partition,
         partition_key,
-        create_time,
-        key.to_s.bytesize + value.to_s.bytesize
+        create_time
       )
 
       @pending_message_queue.write(message)

data/lib/fluent/plugin/out_kafka2.rb

@@ -101,6 +101,16 @@ DESC
       end
       @formatter_proc = setup_formatter(formatter_conf)
 
+      if @default_topic.nil?
+        if @chunk_keys.include?('topic') && !@chunk_keys.include?('tag')
+          log.warn "Use 'topic' field of event record for topic but no fallback. Recommend to set default_topic or set 'tag' in buffer chunk keys like <buffer topic,tag>"
+        end
+      else
+        if @chunk_keys.include?('tag')
+          log.warn "default_topic is set. Fluentd's event tag is not used for topic"
+        end
+      end
+
       @producer_opts = {max_retries: @max_send_retries, required_acks: @required_acks}
       @producer_opts[:ack_timeout] = @ack_timeout if @ack_timeout
       @producer_opts[:compression_codec] = @compression_codec.to_sym if @compression_codec

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -74,6 +74,9 @@ Add a regular expression to capture ActiveSupport notifications from the Kafka c
 requires activesupport gem - records will be generated under fluent_kafka_stats.**
 DESC
 
+  config_param :monitoring_list, :array, :default => [],
+               :desc => "library to be used to monitor. statsd and datadog are supported"
+
   include Fluent::KafkaPluginUtil::SSLSettings
   include Fluent::KafkaPluginUtil::SaslSettings
 
@@ -167,6 +170,11 @@ DESC
         @router.emit("fluent_kafka_stats.#{event.name}", Time.now.to_i, message)
       end
     end
+
+    @monitoring_list.each { |m|
+      require "kafka/#{m}"
+      log.info "#{m} monitoring started"
+    }
   end
 
   def start

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.6.1
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-25 00:00:00.000000000 Z
+date: 2017-08-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -51,14 +51,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.4.0
+        version: 0.4.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.4.0
+        version: 0.4.1
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement