fluent-plugin-kafka 0.5.6 → 0.5.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 268cd322294a6f01fd118de3ef04c22997602d7c
-  data.tar.gz: 570de398931cf778f147d08738eab20ad78b8a9c
+  metadata.gz: cb704a5814ca89bb71f2a9ead76631881a0c3f9e
+  data.tar.gz: 2b6eb6c80c5060a650f17e97bfbee67b478d0e79
 SHA512:
-  metadata.gz: 8e81f53b95cb3ad41c9cb46779ba941021de37ae667f3d7d2be7d24b80e6f1603de95ba673e0bb86d9c4ec54a7b34243b5201cdfc35b7598b2eaa4807221b066
-  data.tar.gz: f804bd9ea046af9dcc4e66081af471100fdcff044add13c0fd26bc3fe1c5241a6b758102b8a55649ef01c0fd0875d2bc1e2f0d87feca6f90b4e5959222e654a3
+  metadata.gz: b2e958ccbc9af8a9b0e4cffc87ad54054d5a579f584ec820a09eb02bcf9392c394938b3c7ad276a2a15b27f8d3d137c00b5f089566b2a03c5bcd0939528b6a68
+  data.tar.gz: 96ac9c4cf08269eef363677dc7dd38a316b54c82b5ca1504661770e210c7dd2c52bb1d7e1edbbeaa3a156b78e332bbe3170bc271ce200b2a46fa964beb59477b

data/ChangeLog

@@ -1,3 +1,7 @@
+Release 0.5.7 - 2017/07/13
+
+	* out_kafka_buffered: Add kafka_agg_max_messages parameter
+
 Release 0.5.6 - 2017/07/10
 
 	* output: Add ActiveSupport notification support

data/README.md

@@ -141,6 +141,8 @@ This plugin uses ruby-kafka producer for writing data. This plugin works with re
       required_acks                (integer)     :default => -1
       ack_timeout                  (integer)     :default => nil (Use default of ruby-kafka)
       compression_codec            (gzip|snappy) :default => nil (No compression)
+      kafka_agg_max_bytes          (integer)     :default => 4096
+      kafka_agg_max_messages       (integer)     :default => nil (No limit)
       max_send_limit_bytes         (integer)     :default => nil (No drop)
       discard_kafka_delivery_failed   (bool)        :default => false (No discard)
     </match>
@@ -157,6 +159,8 @@ Supports following ruby-kafka's producer options.
 - required_acks - default: -1 - The number of acks required per request. If you need flush performance, set lower value, e.g. 1, 2.
 - ack_timeout - default: nil - How long the producer waits for acks. The unit is seconds.
 - compression_codec - default: nil - The codec the producer uses to compress messages.
+- kafka_agg_max_bytes - default: 4096 - Maximum value of total message size to be included in one batch transmission.
+- kafka_agg_max_messages - default: nil - Maximum number of messages to include in one batch transmission.
 - max_send_limit_bytes - default: nil - Max byte size to send message to avoid MessageSizeTooLarge. For example, if you set 1000000(message.max.bytes in kafka), Message more than 1000000 byes will be dropped.
 - discard_kafka_delivery_failed - default: false - discard the record where [Kafka::DeliveryFailed](http://www.rubydoc.info/gems/ruby-kafka/Kafka/DeliveryFailed) occurred
 

data/fluent-plugin-kafka.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.5.6'
+  gem.version       = '0.5.7'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -48,6 +48,7 @@ Set true to remove topic name key from data
 DESC
 
   config_param :kafka_agg_max_bytes, :size, :default => 4*1024  #4k
+  config_param :kafka_agg_max_messages, :integer, :default => nil
   config_param :get_kafka_client_log, :bool, :default => false
 
   # ruby-kafka producer options
@@ -287,8 +288,8 @@ DESC
           next
         end
 
-        if (messages > 0) and (messages_bytes + record_buf_bytes > @kafka_agg_max_bytes)
-          log.debug { "#{messages} messages send because reaches kafka_agg_max_bytes" }
+        if (messages > 0) and (messages_bytes + record_buf_bytes > @kafka_agg_max_bytes) or (@kafka_agg_max_messages && messages >= @kafka_agg_max_messages)
+          log.debug { "#{messages} messages send because reaches the limit of batch transmission." }
           deliver_messages(producer, tag)
           messages = 0
           messages_bytes = 0

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.5.6
+  version: 0.5.7
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-10 00:00:00.000000000 Z
+date: 2017-07-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd