fluent-plugin-kafka 0.12.3 → 0.12.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a5c5fbae81fab15c17ce05013a3a26d924b91cb58c5be753ffdd2c87733667c6
-  data.tar.gz: fc40bae1a4d1c391c9da98c52e442065f2291e7899a8ab20eb4067181efd6a68
+  metadata.gz: 8b9d681257730f610d10ef3f3ab0f219d3167df23482370bcc3c89b01cf12098
+  data.tar.gz: f628e31d41fdc36f51d93a4fb4f7e68321ad0857765a6088a9b46ce593c435c8
 SHA512:
-  metadata.gz: f5df728c0e3516ab4548ad139972fa79729dd5f25d4c5f94423217110f668f15d65fb2d191c6231f446775874ffd2e6b834ad678b8587c1bf47cede4248587f4
-  data.tar.gz: 8a88fc87aef958ec0c43400a6a67e25a01475a607e9d4f38e4da39ac5a926cc2a29199fc911c1258a619a8ca844a2e32d0e2755e8f1c029056fb3e1fb66794d7
+  metadata.gz: 57b996881b2981fb4366c31f25f27b4ca19b4d8ca2c4e27227ac4b5d603ae7a6b2719e7a049d8b668ace11620786832b7e970c90c3536de1ed6ba3da1723dd09
+  data.tar.gz: 21ddc3ab9a074b3910719ae59cdf826423f3cd52c1b9179ab86ccdf6db60166776cd3b7218f6e04da1ac101dd130c87486e30658736e7f6347b161776c7bc7ee

data/ChangeLog

@@ -1,3 +1,7 @@
+Release 0.12.4 - 2020/03/03
+
+	* output: Follow rdkafka log level
+
 Release 0.12.3 - 2020/02/06
 
 	* output: Show warning message for v0.12 plugins

data/README.md

@@ -140,7 +140,7 @@ Consuming topic name is used for event tag. So when the target topic name is `ap
 
 ### Output plugin
 
-This plugin is for fluentd v1.0 or later. This will be `out_kafka` plugin in the future.
+This `kafka2` plugin is for fluentd v1.0 or later. This will be `out_kafka` plugin in the future.
 
     <match app.**>
       @type kafka2
@@ -155,6 +155,8 @@ This plugin is for fluentd v1.0 or later. This will be `out_kafka` plugin in the
       default_message_key   (string) :default => nil
       exclude_topic_key     (bool)   :default => false
       exclude_partition_key (bool)   :default => false
+      exclude_partition     (bool)   :default => false
+      exclude_message_key   (bool)   :default => false
       get_kafka_client_log  (bool)   :default => false
       headers               (hash)   :default => {}
       headers_from_record   (hash)   :default => {}
@@ -197,8 +199,6 @@ Supports following ruby-kafka's producer options.
 - required_acks - default: -1 - The number of acks required per request. If you need flush performance, set lower value, e.g. 1, 2.
 - ack_timeout - default: nil - How long the producer waits for acks. The unit is seconds.
 - compression_codec - default: nil - The codec the producer uses to compress messages.
-- kafka_agg_max_bytes - default: 4096 - Maximum value of total message size to be included in one batch transmission.
-- kafka_agg_max_messages - default: nil - Maximum number of messages to include in one batch transmission.
 - max_send_limit_bytes - default: nil - Max byte size to send message to avoid MessageSizeTooLarge. For example, if you set 1000000(message.max.bytes in kafka), Message more than 1000000 byes will be dropped.
 - discard_kafka_delivery_failed - default: false - discard the record where [Kafka::DeliveryFailed](http://www.rubydoc.info/gems/ruby-kafka/Kafka/DeliveryFailed) occurred
 - monitoring_list - default: [] - library to be used to monitor. statsd and datadog are supported
@@ -292,6 +292,10 @@ Support of fluentd v0.12 has ended. `kafka_buffered` will be an alias of `kafka2
       default_topic         (string) :default => nil
       default_partition_key (string) :default => nil
       default_message_key   (string) :default => nil
+      exclude_topic_key     (bool)   :default => false
+      exclude_partition_key (bool)   :default => false
+      exclude_partition     (bool)   :default => false
+      exclude_message_key   (bool)   :default => false
       output_data_type      (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
       output_include_tag    (bool) :default => false
       output_include_time   (bool) :default => false
@@ -315,6 +319,11 @@ Support of fluentd v0.12 has ended. `kafka_buffered` will be an alias of `kafka2
       monitoring_list              (array)   :default => []
     </match>
 
+`kafka_buffered` has two additional parameters:
+
+- kafka_agg_max_bytes - default: 4096 - Maximum value of total message size to be included in one batch transmission.
+- kafka_agg_max_messages - default: nil - Maximum number of messages to include in one batch transmission.
+
 ### Non-buffered output plugin
 
 This plugin uses ruby-kafka producer for writing data. For performance and reliability concerns, use `kafka_bufferd` output instead. This is mainly for testing.
@@ -349,10 +358,10 @@ This plugin also supports ruby-kafka related parameters. See Buffered output plu
 
 ### rdkafka based output plugin
 
-This plugin uses `rdkafka` instead of `ruby-kafka` for ruby client.
+This plugin uses `rdkafka` instead of `ruby-kafka` for kafka client.
 You need to install rdkafka gem.
 
-    # rdkafka is C extension library so need development tools like ruby-devel, gcc and etc
+    # rdkafka is C extension library. Need to install development tools like ruby-devel, gcc and etc
     # for v0.12 or later
     $ gem install rdkafka --no-document
     # for v0.11 or earlier

data/fluent-plugin-kafka.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.12.3'
+  gem.version       = '0.12.4'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/out_rdkafka.rb

@@ -1,4 +1,5 @@
 require 'thread'
+require 'logger'
 require 'fluent/output'
 require 'fluent/plugin/kafka_plugin_util'
 
@@ -91,8 +92,22 @@ DESC
     super
     log.instance_eval {
       def add(level, &block)
-        if block
+        return unless block
+
+        # Follow rdkakfa's log level. See also rdkafka-ruby's bindings.rb: https://github.com/appsignal/rdkafka-ruby/blob/e5c7261e3f2637554a5c12b924be297d7dca1328/lib/rdkafka/bindings.rb#L117
+        case level
+        when Logger::FATAL
+          self.fatal(block.call)
+        when Logger::ERROR
+          self.error(block.call)
+        when Logger::WARN
+          self.warn(block.call)
+        when Logger::INFO
           self.info(block.call)
+        when Logger::DEBUG
+          self.debug(block.call)
+        else
+          self.trace(block.call)
         end
       end
     }

data/lib/fluent/plugin/out_rdkafka2.rb

@@ -1,4 +1,5 @@
 require 'thread'
+require 'logger'
 require 'fluent/plugin/output'
 require 'fluent/plugin/kafka_plugin_util'
 
@@ -108,8 +109,22 @@ DESC
       super
       log.instance_eval {
         def add(level, &block)
-          if block
+          return unless block
+
+          # Follow rdkakfa's log level. See also rdkafka-ruby's bindings.rb: https://github.com/appsignal/rdkafka-ruby/blob/e5c7261e3f2637554a5c12b924be297d7dca1328/lib/rdkafka/bindings.rb#L117
+          case level
+          when Logger::FATAL
+            self.fatal(block.call)
+          when Logger::ERROR
+            self.error(block.call)
+          when Logger::WARN
+            self.warn(block.call)
+          when Logger::INFO
             self.info(block.call)
+          when Logger::DEBUG
+            self.debug(block.call)
+          else
+            self.trace(block.call)
           end
         end
       }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.12.3
+  version: 0.12.4
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-06 00:00:00.000000000 Z
+date: 2020-03-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd