fluent-plugin-kafka 0.3.2 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 512f9e6a1ab34f3206a79904ff3134da1ee6de54
-  data.tar.gz: 27bcf99b9a4eac5ed2cf15694a249db4f3db6ffb
+  metadata.gz: 0dc35a4ee0bdd32fef9268ffb4a89a163fd7d78a
+  data.tar.gz: 64db45e59ff24473f1adabde25bc2a65f237fbb6
 SHA512:
-  metadata.gz: 0d926b09af1ed8611d85a40eb353aeda21b39f862c85a4131ada9e9fcfa9d057157fb107a56808af905c89c669c6a452b03eef508e665ae3957f7599c25c19af
-  data.tar.gz: 60d029566a17521e19dd93e0e6d2cefd35990a8b14d89202c42ba6cf6512608f78dced06e23d0e1386e95d206173ad78a77c65dba081f1c3e5adc150f74c07d4
+  metadata.gz: 58fdebb9585c65e30b290d3f72707046942873e86220fdc4207c6d3194faca3cf015c487d71df2f61c491a9089210b35c2015a4d0ba639f5acfa30b1250cc46f
+  data.tar.gz: 6c331f30a9c19d58f66b916b7ef01f85a999121710e34eb6352db0f4a43b08af436ca017fd56e09b5a9a6ef825ffd92cadbb900d99c88a3972aa6fe7dcb7a13a

data/README.md

@@ -173,6 +173,7 @@ This plugin uses ruby-kafka producer for writing data. This plugin works with re
       output_data_type      (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
       output_include_tag    (bool) :default => false
       output_include_time   (bool) :default => false
+      get_kafka_client_log  (bool) :default => false
 
       # See fluentd document for buffer related parameters: http://docs.fluentd.org/articles/buffer-plugin-overview
 
@@ -183,6 +184,12 @@ This plugin uses ruby-kafka producer for writing data. This plugin works with re
       compression_codec   (gzip|snappy) :default => nil (No compression)
     </match>
 
+`<formatter name>` of `output_data_type` uses fluentd's formatter plugins. See [formatter article](http://docs.fluentd.org/articles/formatter-plugin-overview).
+
+ruby-kafka sometimes returns `Kafka::DeliveryFailed` error without good information.
+In this case, `get_kafka_client_log` is useful for identifying the error cause.    
+ruby-kafka's log is routed to fluentd log so you can see ruby-kafka's log in fluentd logs.
+
 Supports following ruby-kafka's producer options.
 
 - max_send_retries - default: 1 - Number of times to retry sending of messages to a leader.

data/fluent-plugin-kafka.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.3.2'
+  gem.version       = '0.3.3'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/in_kafka_group.rb

@@ -18,6 +18,8 @@ class Fluent::KafkaGroupInput < Fluent::Input
                :desc => "Tag prefix (Optional)"
   config_param :add_suffix, :string, :default => nil,
                :desc => "Tag suffix (Optional)"
+  config_param :retry_emit_limit, :integer, :default => nil,
+               :desc => "How long to stop event consuming when BufferQueueLimitError happens. Wait retry_emit_limit x 1s. The default is waiting until BufferQueueLimitError is resolved"
 
   # Kafka consumer options
   config_param :max_wait_time, :integer, :default => nil,
@@ -35,6 +37,9 @@ class Fluent::KafkaGroupInput < Fluent::Input
 
   include Fluent::KafkaPluginUtil::SSLSettings
 
+  class ForShutdown < StandardError
+  end
+
   unless method_defined?(:router)
     define_method("router") { Fluent::Engine }
   end
@@ -146,9 +151,10 @@ class Fluent::KafkaGroupInput < Fluent::Input
           }
 
           unless es.empty?
-            router.emit_stream(tag, es)
+            emit_events(tag, es)
           end
         }
+      rescue ForShutdown
       rescue => e
         log.error "unexpected error during consuming events from kafka. Re-fetch events.", :error => e.to_s
         log.error_backtrace
@@ -158,4 +164,26 @@ class Fluent::KafkaGroupInput < Fluent::Input
     log.error "unexpected error during consumer object access", :error => e.to_s
     log.error_backtrace
   end
+
+  def emit_events(tag, es)
+    retries = 0
+    begin
+      router.emit_stream(tag, es)
+    rescue Fluent::BufferQueueLimitError
+      raise ForShutdown if @consumer.nil?
+
+      if @retry_emit_limit.nil?
+        sleep 1
+        retry
+      end
+
+      if retries < @retry_emit_limit
+        retries += 1
+        sleep 1
+        retry
+      else
+        raise RuntimeError, "Exceeds retry_emit_limit"
+      end
+    end
+  end
 end

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -29,6 +29,7 @@ DESC
   config_param :output_include_tag, :bool, :default => false
   config_param :output_include_time, :bool, :default => false
   config_param :kafka_agg_max_bytes, :size, :default => 4*1024  #4k
+  config_param :get_kafka_client_log, :bool, :default => false
 
   # ruby-kafka producer options
   config_param :max_send_retries, :integer, :default => 1,
@@ -78,7 +79,8 @@ DESC
     end
     begin
       if @seed_brokers.length > 0
-        @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+        logger = @get_kafka_client_log ? log : nil
+        @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                            ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key))
         log.info "initialized kafka producer: #{@client_id}"
       else
@@ -196,25 +198,34 @@ DESC
     bytes_by_topic = {}
     messages = 0
     messages_bytes = 0
+    record_buf = nil
+    record_buf_bytes = nil
+
     begin
       chunk.msgpack_each { |time, record|
-        if @output_include_time
-          if @time_format
-            record['time'.freeze] = Time.at(time).strftime(@time_format)
-          else
-            record['time'.freeze] = time
+        begin
+          if @output_include_time
+            if @time_format
+              record['time'.freeze] = Time.at(time).strftime(@time_format)
+            else
+              record['time'.freeze] = time
+            end
           end
-        end
 
-        record['tag'] = tag if @output_include_tag
-        topic = record['topic'.freeze] || def_topic
-        partition_key = record['partition_key'.freeze] || @default_partition_key
+          record['tag'] = tag if @output_include_tag
+          topic = record['topic'.freeze] || def_topic
+          partition_key = record['partition_key'.freeze] || @default_partition_key
 
-        records_by_topic[topic] ||= 0
-        bytes_by_topic[topic] ||= 0
+          records_by_topic[topic] ||= 0
+          bytes_by_topic[topic] ||= 0
+
+          record_buf = @formatter_proc.call(tag, time, record)
+          record_buf_bytes = record_buf.bytesize
+        rescue StandardError => e
+          log.warn "unexpected error during format record. Skip broken event:", :error => e.to_s, :error_class => e.class.to_s, :time => time, :record => record
+          next
+        end
 
-        record_buf = @formatter_proc.call(tag, time, record)
-        record_buf_bytes = record_buf.bytesize
         if (messages > 0) and (messages_bytes + record_buf_bytes > @kafka_agg_max_bytes)
           log.on_trace { log.trace("#{messages} messages send.") }
           producer.deliver_messages
@@ -237,6 +248,7 @@ DESC
     end
   rescue Exception => e
     log.warn "Send exception occurred: #{e}"
+    log.warn "Exception Backtrace : #{e.backtrace.join("\n")}"
     # For safety, refresh client and its producers
     shutdown_producers
     refresh_client(false)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.3.3
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-06 00:00:00.000000000 Z
+date: 2016-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd