fluent-plugin-kafka 0.5.3 → 0.5.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 196c86d33e2eb43abe78d5a0c89b3b72eb511582
-  data.tar.gz: df09c952dc8fbaa35da53d12e86dd74e25a5824f
+  metadata.gz: 891e2173b1a15ec7e20c6793aa9fd05dae0fdfbc
+  data.tar.gz: 0f810a44129c6a274eb0b486a86c09d5e723b68d
 SHA512:
-  metadata.gz: 40e4f0f6333983678bf08521de4ec800ac5158a12b1b6f22e7bc2b0d19bd88b4bcc2ca8c18e49bcf72c3af300ee6902c101310efc83b9d9a2acc27c32dfeab0f
-  data.tar.gz: 591ebc8a337b9b41e8c7b2d066539b2a17cd0ae041630ff8ec9d9555b1ea27bdef5691b21805ab15fd86c2616649510ac11877f5040077b27c5be07328fdb6d3
+  metadata.gz: '09895588da32e7b8d7faf12dcae9b0fe76f7ab038c811b2674a588d88853e69e55e1ede7976a585250136bb8eacc522455b6a45ba2c555cf61638bd91a6e0bcb'
+  data.tar.gz: c9ce862ec1619447d67f204c3266b9b74491a47c43d4caa25e3bba111e8c58f18ce7d0973f5915e75ead3fa37170246b841698583376179b14345e4335a57366

data/ChangeLog

@@ -1,3 +1,8 @@
+Release 0.5.4 - 2017/04/12
+
+	* out_kafka_buffered: Add max_send_limit_bytes parameter
+	* out_kafka: Improve buffer overflow handling of ruby-kafka
+
 Release 0.5.3 - 2017/02/13
 
 	* Relax ruby-kafka dependency

data/README.md

@@ -137,10 +137,11 @@ This plugin uses ruby-kafka producer for writing data. This plugin works with re
       # See fluentd document for buffer related parameters: http://docs.fluentd.org/articles/buffer-plugin-overview
 
       # ruby-kafka producer options
-      max_send_retries    (integer)     :default => 1
-      required_acks       (integer)     :default => -1
-      ack_timeout         (integer)     :default => nil (Use default of ruby-kafka)
-      compression_codec   (gzip|snappy) :default => nil (No compression)
+      max_send_retries     (integer)     :default => 1
+      required_acks        (integer)     :default => -1
+      ack_timeout          (integer)     :default => nil (Use default of ruby-kafka)
+      compression_codec    (gzip|snappy) :default => nil (No compression)
+      max_send_limit_bytes (integer)     :default => nil (No drop)
     </match>
 
 `<formatter name>` of `output_data_type` uses fluentd's formatter plugins. See [formatter article](http://docs.fluentd.org/articles/formatter-plugin-overview).
@@ -155,6 +156,7 @@ Supports following ruby-kafka's producer options.
 - required_acks - default: -1 - The number of acks required per request. If you need flush performance, set lower value, e.g. 1, 2.
 - ack_timeout - default: nil - How long the producer waits for acks. The unit is seconds.
 - compression_codec - default: nil - The codec the producer uses to compress messages.
+- max_send_limit_bytes - default: nil - Max byte size to send message to avoid MessageSizeTooLarge. For example, if you set 1000000(message.max.bytes in kafka), Message more than 1000000 byes will be dropped.
 
 See also [Kafka::Client](http://www.rubydoc.info/gems/ruby-kafka/Kafka/Client) for more detailed documentation about ruby-kafka.
 
@@ -204,10 +206,12 @@ This plugin uses ruby-kafka producer for writing data. For performance and relia
       exclude_partition_key (bool) :default => false
 
       # ruby-kafka producer options
-      max_send_retries  (integer)     :default => 1
-      required_acks     (integer)     :default => -1
-      ack_timeout       (integer)     :default => nil (Use default of ruby-kafka)
-      compression_codec (gzip|snappy) :default => nil
+      max_send_retries    (integer)     :default => 1
+      required_acks       (integer)     :default => -1
+      ack_timeout         (integer)     :default => nil (Use default of ruby-kafka) 
+      compression_codec   (gzip|snappy) :default => nil
+      max_buffer_size     (integer)     :default => nil (Use default of ruby-kafka)
+      max_buffer_bytesize (integer)     :default => nil (Use default of ruby-kafka) 
     </match>
 
 This plugin also supports ruby-kafka related parameters. See Buffered output plugin section.

data/fluent-plugin-kafka.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.5.3'
+  gem.version       = '0.5.4'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/out_kafka.rb

@@ -54,6 +54,12 @@ DESC
 
   config_param :time_format, :string, :default => nil
 
+  config_param :max_buffer_size, :integer, :default => nil,
+               :desc => "Number of messages to be buffered by the kafka producer."
+
+  config_param :max_buffer_bytesize, :integer, :default => nil,
+               :desc => "Maximum size in bytes to be buffered."
+
   include Fluent::KafkaPluginUtil::SSLSettings
 
   attr_accessor :output_data_type
@@ -122,6 +128,8 @@ DESC
     @producer_opts = {max_retries: @max_send_retries, required_acks: @required_acks}
     @producer_opts[:ack_timeout] = @ack_timeout if @ack_timeout
     @producer_opts[:compression_codec] = @compression_codec.to_sym if @compression_codec
+    @producer_opts[:max_buffer_size] = @max_buffer_size if @max_buffer_size
+    @producer_opts[:max_buffer_bytesize] = @max_buffer_bytesize if @max_buffer_bytesize
   end
 
   def start
@@ -183,8 +191,15 @@ DESC
 
         value = @formatter_proc.call(tag, time, record)
 
-        log.on_trace { log.trace("message will send to #{topic} with partition_key: #{partition_key}, partition: #{partition}, message_key: #{message_key} and value: #{record_buf}.") }
-        producer.produce(value, topic: topic, key: message_key, partition: partition, partition_key: partition_key)
+        log.on_trace { log.trace("message will send to #{topic} with partition_key: #{partition_key}, partition: #{partition}, message_key: #{message_key} and value: #{value}.") }
+	begin
+          producer.produce(value, topic: topic, key: message_key, partition: partition, partition_key: partition_key)
+	rescue Kafka::BufferOverflow => e
+	  log.warn "BufferOverflow occurred: #{e}"
+	  log.info "Trying to deliver the messages to prevent the buffer from overflowing again."
+	  producer.deliver_messages
+	  log.info "Recovered from BufferOverflow successfully`"
+	end
       end
 
       producer.deliver_messages

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -62,6 +62,7 @@ DESC
 The codec the producer uses to compress messages.
 Supported codecs: (gzip|snappy)
 DESC
+  config_param :max_send_limit_bytes, :size, :default => nil
 
   config_param :time_format, :string, :default => nil
 
@@ -242,6 +243,10 @@ DESC
 
           record_buf = @formatter_proc.call(tag, time, record)
           record_buf_bytes = record_buf.bytesize
+          if @max_send_limit_bytes && record_buf_bytes > @max_send_limit_bytes
+            log.warn "record size exceeds max_send_limit_bytes. Skip event:", :time => time, :record => record
+            next
+          end
         rescue StandardError => e
           log.warn "unexpected error during format record. Skip broken event:", :error => e.to_s, :error_class => e.class.to_s, :time => time, :record => record
           next

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: 0.5.4
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-13 00:00:00.000000000 Z
+date: 2017-04-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -131,7 +131,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin for Apache Kafka > 0.8