fluent-plugin-kafka 0.5.4 → 0.5.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 891e2173b1a15ec7e20c6793aa9fd05dae0fdfbc
-  data.tar.gz: 0f810a44129c6a274eb0b486a86c09d5e723b68d
+  metadata.gz: 25a3a2247250b5f3c839378914e21ff82906986f
+  data.tar.gz: e1f643b5798a8b07f12f7874391e449c5cd16f05
 SHA512:
-  metadata.gz: '09895588da32e7b8d7faf12dcae9b0fe76f7ab038c811b2674a588d88853e69e55e1ede7976a585250136bb8eacc522455b6a45ba2c555cf61638bd91a6e0bcb'
-  data.tar.gz: c9ce862ec1619447d67f204c3266b9b74491a47c43d4caa25e3bba111e8c58f18ce7d0973f5915e75ead3fa37170246b841698583376179b14345e4335a57366
+  metadata.gz: 3dae82784298f2b431568805c5b4955cb64bb11e0e09f41bb95068c58f815fbd0b370dc5262a3254e196c28a08cf77f822c9b6ad91d2829901ef4983b4e80424
+  data.tar.gz: 589d4ad5267d694732a2e5cf93d669516ea21f25d1d6872c4410ecd652fdd6cb24319e19db0ebf3aaba5639e44743c3c6fd4579c9d3e2fd1f61e6bb463a59df3

data/.travis.yml

@@ -4,6 +4,7 @@ rvm:
   - 2.1
   - 2.2
   - 2.3.1
+  - 2.4.1
   - ruby-head
 
 script:

data/ChangeLog

@@ -1,3 +1,8 @@
+Release 0.5.5 - 2017/04/19
+
+	* output: Some trace log level changed to debug
+	* out_kafka_buffered: Add discard_kafka_delivery_failed parameter
+
 Release 0.5.4 - 2017/04/12
 
 	* out_kafka_buffered: Add max_send_limit_bytes parameter

data/README.md

@@ -137,11 +137,12 @@ This plugin uses ruby-kafka producer for writing data. This plugin works with re
       # See fluentd document for buffer related parameters: http://docs.fluentd.org/articles/buffer-plugin-overview
 
       # ruby-kafka producer options
-      max_send_retries     (integer)     :default => 1
-      required_acks        (integer)     :default => -1
-      ack_timeout          (integer)     :default => nil (Use default of ruby-kafka)
-      compression_codec    (gzip|snappy) :default => nil (No compression)
-      max_send_limit_bytes (integer)     :default => nil (No drop)
+      max_send_retries             (integer)     :default => 1
+      required_acks                (integer)     :default => -1
+      ack_timeout                  (integer)     :default => nil (Use default of ruby-kafka)
+      compression_codec            (gzip|snappy) :default => nil (No compression)
+      max_send_limit_bytes         (integer)     :default => nil (No drop)
+      discard_kafka_delivery_failed   (bool)        :default => false (No discard)
     </match>
 
 `<formatter name>` of `output_data_type` uses fluentd's formatter plugins. See [formatter article](http://docs.fluentd.org/articles/formatter-plugin-overview).
@@ -157,6 +158,7 @@ Supports following ruby-kafka's producer options.
 - ack_timeout - default: nil - How long the producer waits for acks. The unit is seconds.
 - compression_codec - default: nil - The codec the producer uses to compress messages.
 - max_send_limit_bytes - default: nil - Max byte size to send message to avoid MessageSizeTooLarge. For example, if you set 1000000(message.max.bytes in kafka), Message more than 1000000 byes will be dropped.
+- discard_kafka_delivery_failed - default: false - discard the record where [Kafka::DeliveryFailed](http://www.rubydoc.info/gems/ruby-kafka/Kafka/DeliveryFailed) occurred
 
 See also [Kafka::Client](http://www.rubydoc.info/gems/ruby-kafka/Kafka/Client) for more detailed documentation about ruby-kafka.
 

data/fluent-plugin-kafka.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.5.4'
+  gem.version       = '0.5.5'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/out_kafka.rb

@@ -191,7 +191,7 @@ DESC
 
         value = @formatter_proc.call(tag, time, record)
 
-        log.on_trace { log.trace("message will send to #{topic} with partition_key: #{partition_key}, partition: #{partition}, message_key: #{message_key} and value: #{value}.") }
+        log.trace { "message will send to #{topic} with partition_key: #{partition_key}, partition: #{partition}, message_key: #{message_key} and value: #{value}." }
 	begin
           producer.produce(value, topic: topic, key: message_key, partition: partition, partition_key: partition_key)
 	rescue Kafka::BufferOverflow => e

data/lib/fluent/plugin/out_kafka2.rb

@@ -162,14 +162,14 @@ DESC
             next
           end
 
-          log.on_trace { "message will send to #{topic} with partition_key: #{partition_key}, partition: #{partition}, message_key: #{message_key} and value: #{record_buf}." }
+          log.trace { "message will send to #{topic} with partition_key: #{partition_key}, partition: #{partition}, message_key: #{message_key} and value: #{record_buf}." }
           messages += 1
 
           producer.produce(record_buf, message_key, partition, partition_key)
         }
 
         if messages > 0
-          log.trace { "#{messages} messages send." }
+          log.debug { "#{messages} messages send." }
           producer.deliver_messages
         end
       end

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -63,6 +63,7 @@ The codec the producer uses to compress messages.
 Supported codecs: (gzip|snappy)
 DESC
   config_param :max_send_limit_bytes, :size, :default => nil
+  config_param :discard_kafka_delivery_failed, :bool, :default => false
 
   config_param :time_format, :string, :default => nil
 
@@ -142,6 +143,11 @@ DESC
     @producer_opts = {max_retries: @max_send_retries, required_acks: @required_acks}
     @producer_opts[:ack_timeout] = @ack_timeout if @ack_timeout
     @producer_opts[:compression_codec] = @compression_codec.to_sym if @compression_codec
+
+    if @discard_kafka_delivery_failed
+      log.warn "'discard_kafka_delivery_failed' option discards events which cause delivery failure, e.g. invalid topic or something."
+      log.warn "If this is unexpected, you need to check your configuration or data."
+    end
   end
 
   def start
@@ -209,6 +215,19 @@ DESC
     end
   end
 
+  def deliver_messages(producer, tag)
+    if @discard_kafka_delivery_failed
+      begin
+        producer.deliver_messages
+      rescue Kafka::DeliveryFailed => e
+        log.warn "DeliveryFailed occurred. Discard broken event:", :error => e.to_s, :error_class => e.class.to_s, :tag => tag
+        producer.clear_buffer
+      end
+    else
+      producer.deliver_messages
+    end
+  end
+
   def write(chunk)
     tag = chunk.key
     def_topic = @default_topic || tag
@@ -253,12 +272,12 @@ DESC
         end
 
         if (messages > 0) and (messages_bytes + record_buf_bytes > @kafka_agg_max_bytes)
-          log.on_trace { log.trace("#{messages} messages send.") }
-          producer.deliver_messages
+          log.debug { "#{messages} messages send because reaches kafka_agg_max_bytes" }
+          deliver_messages(producer, tag)
           messages = 0
           messages_bytes = 0
         end
-        log.on_trace { log.trace("message will send to #{topic} with partition_key: #{partition_key}, partition: #{partition}, message_key: #{message_key} and value: #{record_buf}.") }
+        log.trace { "message will send to #{topic} with partition_key: #{partition_key}, partition: #{partition}, message_key: #{message_key} and value: #{record_buf}." }
         messages += 1
         producer.produce2(record_buf, topic: topic, key: message_key, partition_key: partition_key, partition: partition)
         messages_bytes += record_buf_bytes
@@ -267,8 +286,8 @@ DESC
         bytes_by_topic[topic] += record_buf_bytes
       }
       if messages > 0
-        log.trace { "#{messages} messages send." }
-        producer.deliver_messages
+        log.debug { "#{messages} messages send." }
+        deliver_messages(producer, tag)
       end
       log.debug { "(records|bytes) (#{records_by_topic}|#{bytes_by_topic})" }
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.5.4
+  version: 0.5.5
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-04-12 00:00:00.000000000 Z
+date: 2017-04-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd