fluent-plugin-kafka 0.3.3 → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0dc35a4ee0bdd32fef9268ffb4a89a163fd7d78a
-  data.tar.gz: 64db45e59ff24473f1adabde25bc2a65f237fbb6
+  metadata.gz: e2d81baf9816d05b5cc5f5c982aafe722781d978
+  data.tar.gz: cfb1477ba8cd4961424881c173cf0d4dffab999d
 SHA512:
-  metadata.gz: 58fdebb9585c65e30b290d3f72707046942873e86220fdc4207c6d3194faca3cf015c487d71df2f61c491a9089210b35c2015a4d0ba639f5acfa30b1250cc46f
-  data.tar.gz: 6c331f30a9c19d58f66b916b7ef01f85a999121710e34eb6352db0f4a43b08af436ca017fd56e09b5a9a6ef825ffd92cadbb900d99c88a3972aa6fe7dcb7a13a
+  metadata.gz: 15b62cb6289281bebf2938172b41fe0a8f7080d6afebfebf672e45a10a1c31779db3ed9b739a28db57300ed5323abd814ae69720e0afc0d3a301ffebfce66e8b
+  data.tar.gz: 70deeef4310ef4ad2674ba21fb2f981147bc0784810d88c14ab32efe837f7b1f7fc41f6bb540223d65bc1df0cae599440ae05eaaae7ded3185c6865a727fedbb

data/README.md

@@ -116,6 +116,8 @@ This plugin uses ruby-kafka producer for writing data. For performance and relia
       output_data_type      (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
       output_include_tag    (bool) :default => false
       output_include_time   (bool) :default => false
+      exclude_topic_key     (bool) :default => false
+      exclude_partition_key (bool) :default => false
 
       # ruby-kafka producer options
       max_send_retries  (integer)     :default => 1
@@ -173,6 +175,8 @@ This plugin uses ruby-kafka producer for writing data. This plugin works with re
       output_data_type      (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
       output_include_tag    (bool) :default => false
       output_include_time   (bool) :default => false
+      exclude_topic_key     (bool) :default => false
+      exclude_partition_key (bool) :default => false
       get_kafka_client_log  (bool) :default => false
 
       # See fluentd document for buffer related parameters: http://docs.fluentd.org/articles/buffer-plugin-overview

data/fluent-plugin-kafka.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.3.3'
+  gem.version       = '0.3.4'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/out_kafka.rb

@@ -22,9 +22,17 @@ DESC
                :desc => "Supported format: (json|ltsv|msgpack|attr:<record name>|<formatter name>)"
   config_param :output_include_tag, :bool, :default => false
   config_param :output_include_time, :bool, :default => false
+  config_param :exclude_partition_key, :bool, :default => false,
+                       :desc => <<-DESC
+Set true to remove partition key from data
+DESC
+  config_param :exclude_topic_key, :bool, :default => false,
+                         :desc => <<-DESC
+Set true to remove topic name key from data
+DESC
 
   # ruby-kafka producer options
-  config_param :max_send_retries, :integer, :default => 1,
+  config_param :max_send_retries, :integer, :default => 2,
                :desc => "Number of times to retry sending of messages to a leader."
   config_param :required_acks, :integer, :default => -1,
                :desc => "The number of acks required per request."
@@ -157,8 +165,9 @@ DESC
           end
         end
         record['tag'] = tag if @output_include_tag
-        topic = record['topic'] || @default_topic || tag
-        partition_key = record['partition_key'] || @default_partition_key
+        topic = (@exclude_topic_key ? record.delete('topic') : record['topic']) || @default_topic || tag
+        partition_key = (@exclude_partition_key ? record.delete('partition_key') : record['partition_key']) || @default_partition_key
+
         value = @formatter_proc.call(tag, time, record)
 
         log.on_trace { log.trace("message send to #{topic} with key: #{partition_key} and value: #{value}.") }

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -28,11 +28,20 @@ Supported format: (json|ltsv|msgpack|attr:<record name>|<formatter name>)
 DESC
   config_param :output_include_tag, :bool, :default => false
   config_param :output_include_time, :bool, :default => false
+  config_param :exclude_partition_key, :bool, :default => false,
+               :desc => <<-DESC
+Set true to remove partition key from data
+DESC
+   config_param :exclude_topic_key, :bool, :default => false,
+                :desc => <<-DESC
+Set true to remove topic name key from data
+DESC
+
   config_param :kafka_agg_max_bytes, :size, :default => 4*1024  #4k
   config_param :get_kafka_client_log, :bool, :default => false
 
   # ruby-kafka producer options
-  config_param :max_send_retries, :integer, :default => 1,
+  config_param :max_send_retries, :integer, :default => 2,
                :desc => "Number of times to retry sending of messages to a leader."
   config_param :required_acks, :integer, :default => -1,
                :desc => "The number of acks required per request."
@@ -213,8 +222,8 @@ DESC
           end
 
           record['tag'] = tag if @output_include_tag
-          topic = record['topic'.freeze] || def_topic
-          partition_key = record['partition_key'.freeze] || @default_partition_key
+          topic = (@exclude_topic_key ? record.delete('topic'.freeze) : record['topic'.freeze]) || def_topic
+          partition_key = (@exclude_partition_key ? record.delete('partition_key'.freeze) : record['partition_key'.freeze]) || @default_partition_key
 
           records_by_topic[topic] ||= 0
           bytes_by_topic[topic] ||= 0

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.3.4
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-17 00:00:00.000000000 Z
+date: 2016-10-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd