fluentd 0.10.40 → 0.10.41

data/ChangeLog

@@ -1,3 +1,9 @@
+Release 0.10.41 - 2013/12/05
+
+* BufferedOutput: Add 'max_retry_wait' to limit retry interval
+* in_exec/object_space/syslog: Improve emit error message
+* Log: Add --suppress-repeated-stacktrace option to prevent log size explosion
+
 Release 0.10.40 - 2013/11/08
 
 * Dump gem version of fluent-plugin-xxx and fluent-mixin-xxx

data/README.rdoc

@@ -1,5 +1,7 @@
 = Fluentd
 
+{<img src="https://travis-ci.org/fluent/fluentd.png" />}[https://travis-ci.org/fluent/fluentd]
+
 Fluentd is an event collector system. It is a generalized version of syslogd, which handles JSON objects for its log messages.
 
 == Architecture

data/lib/fluent/command/fluentd.rb

@@ -36,6 +36,7 @@ opts = {
   :chuser => nil,
   :chgroup => nil,
   :suppress_interval => 0,
+  :suppress_repeated_stacktrace => false,
 }
 
 op.on('-s', "--setup [DIR=#{File.dirname(Fluent::DEFAULT_CONFIG_PATH)}]", "install sample configuration file to the directory") {|s|
@@ -86,6 +87,10 @@ op.on('--emit-error-log-interval SECONDS', "suppress interval seconds of emit er
   opts[:suppress_interval] = s.to_i
 }
 
+op.on('--suppress-repeated-stacktrace', "suppress repeated stacktrace", TrueClass) {|b|
+  opts[:suppress_repeated_stacktrace] = b
+}
+
 op.on('-v', '--verbose', "increase verbose level (-v: debug, -vv: trace)", TrueClass) {|b|
   if b
     opts[:log_level] = [opts[:log_level] - 1, Fluent::Log::LEVEL_TRACE].max

data/lib/fluent/log.rb

@@ -40,7 +40,7 @@ module Fluent
 
     LEVEL_TEXT = %w(trace debug info warn error fatal)
 
-    def initialize(out=STDERR, level=LEVEL_TRACE)
+    def initialize(out=STDERR, level=LEVEL_TRACE, opts={})
       @out = out
       @level = level
       @debug_mode = false
@@ -50,6 +50,10 @@ module Fluent
       enable_color out.tty?
       # TODO: This variable name is unclear so we should change to better name.
       @threads_exclude_events = []
+
+      if opts.has_key?(:suppress_repeated_stacktrace)
+        @suppress_repeated_stacktrace = opts[:suppress_repeated_stacktrace]
+      end
     end
 
     attr_accessor :out
@@ -110,12 +114,7 @@ module Fluent
     alias TRACE trace
 
     def trace_backtrace(backtrace=$!.backtrace)
-      return if @level > LEVEL_TRACE
-      time = Time.now
-      backtrace.each {|msg|
-        puts ["  ", caller_line(time, 4, LEVEL_TRACE), msg].join
-      }
-      nil
+      dump_stacktrace(backtrace, LEVEL_TRACE)
     end
 
     def on_debug(&block)
@@ -133,12 +132,7 @@ module Fluent
     alias DEBUG debug
 
     def debug_backtrace(backtrace=$!.backtrace)
-      return if @level > LEVEL_DEBUG
-      time = Time.now
-      backtrace.each {|msg|
-        puts ["  ", caller_line(time, 4, LEVEL_DEBUG), msg].join
-      }
-      nil
+      dump_stacktrace(backtrace, LEVEL_DEBUG)
     end
 
     def on_info(&block)
@@ -156,12 +150,7 @@ module Fluent
     alias INFO info
 
     def info_backtrace(backtrace=$!.backtrace)
-      return if @level > LEVEL_INFO
-      time = Time.now
-      backtrace.each {|msg|
-        puts ["  ", caller_line(time, 4, LEVEL_INFO), msg].join
-      }
-      nil
+      dump_stacktrace(backtrace, LEVEL_INFO)
     end
 
     def on_warn(&block)
@@ -179,12 +168,7 @@ module Fluent
     alias WARN warn
 
     def warn_backtrace(backtrace=$!.backtrace)
-      return if @level > LEVEL_WARN
-      time = Time.now
-      backtrace.each {|msg|
-        puts ["  ", caller_line(time, 4, LEVEL_WARN), msg].join
-      }
-      nil
+      dump_stacktrace(backtrace, LEVEL_WARN)
     end
 
     def on_error(&block)
@@ -202,12 +186,7 @@ module Fluent
     alias ERROR error
 
     def error_backtrace(backtrace=$!.backtrace)
-      return if @level > LEVEL_ERROR
-      time = Time.now
-      backtrace.each {|msg|
-        puts ["  ", caller_line(time, 4, LEVEL_ERROR), msg].join
-      }
-      nil
+      dump_stacktrace(backtrace, LEVEL_ERROR)
     end
 
     def on_fatal(&block)
@@ -225,12 +204,7 @@ module Fluent
     alias FATAL fatal
 
     def fatal_backtrace(backtrace=$!.backtrace)
-      return if @level > LEVEL_FATAL
-      time = Time.now
-      backtrace.each {|msg|
-        puts ["  ", caller_line(time, 4, LEVEL_FATAL), msg].join
-      }
-      nil
+      dump_stacktrace(backtrace, LEVEL_FATAL)
     end
 
     def puts(msg)
@@ -251,6 +225,23 @@ module Fluent
     end
 
     private
+
+    def dump_stacktrace(backtrace, level)
+      return if @level > level
+
+      time = Time.now
+      if @suppress_repeated_stacktrace && (Thread.current[:last_repeated_stacktrace] == backtrace)
+        puts ["  ", caller_line(time, 5, level), 'suppressed same stacktrace'].join
+      else
+        backtrace.each { |msg|
+          puts ["  ", caller_line(time, 5, level), msg].join
+        }
+        Thread.current[:last_repeated_stacktrace] = backtrace if @suppress_repeated_stacktrace
+      end
+
+      nil
+    end
+
     def event(level, args)
       time = Time.now
       message = ''
@@ -282,9 +273,9 @@ module Fluent
     end
 
     def caller_line(time, depth, level)
-      line = caller(depth+1)[0]
       log_msg = "#{time.strftime(@time_format)}[#{LEVEL_TEXT[level]}]: "
       if @debug_mode
+        line = caller(depth+1)[0]
         if match = /^(.+?):(\d+)(?::in `(.*)')?/.match(line)
           file = match[1].split('/')[-2,2].join('/')
           line = match[2]

data/lib/fluent/output.rb

@@ -174,6 +174,7 @@ module Fluent
     config_param :flush_interval, :time, :default => 60
     config_param :retry_limit, :integer, :default => 17
     config_param :retry_wait, :time, :default => 1.0
+    config_param :max_retry_wait, :time, :default => nil
     config_param :num_threads, :integer, :default => 1
     config_param :queued_chunk_flush_interval, :time, :default => 1
 
@@ -388,7 +389,8 @@ module Fluent
                # secondary retry
                @retry_wait * (2 ** (@error_history.size-2-@retry_limit))
              end
-      wait + (rand * (wait / 4.0) - (wait / 8.0))
+      retry_wait = wait + (rand * (wait / 4.0) - (wait / 8.0))
+      @max_retry_wait ? [retry_wait, @max_retry_wait].min : retry_wait
     end
 
     def write_abort

data/lib/fluent/plugin/in_exec.rb

@@ -133,22 +133,21 @@ module Fluent
     private
 
     def on_message(record)
-      if val = record.delete(@time_key)
-        time = @time_parse_proc.call(val)
-      else
-        time = Engine.now
-      end
-
       if val = record.delete(@tag_key)
         tag = val
       else
         tag = @tag
       end
 
+      if val = record.delete(@time_key)
+        time = @time_parse_proc.call(val)
+      else
+        time = Engine.now
+      end
+
       Engine.emit(tag, time, record)
-    rescue
-      $log.error "exec failed to emit", :error => $!.to_s, :error_class => $!.class.to_s, :record => Yajl.dump(record)
-      $log.warn_backtrace $!.backtrace
+    rescue => e
+      $log.error "exec failed to emit", :error => e.to_s, :error_class => e.class.to_s, :tag => tag, :record => Yajl.dump(record)
     end
   end
 end

data/lib/fluent/plugin/in_object_space.rb

@@ -109,6 +109,8 @@ module Fluent
       }
 
       Engine.emit(@tag, now, record)
+    rescue => e
+      $log.error "object space failed to emit", :error => e.to_s, :error_class => e.class.to_s, :tag => @tag, :record => Yajl.dump(record)
     end
   end
 end

data/lib/fluent/plugin/in_syslog.rb

@@ -177,6 +177,8 @@ module Fluent
       tag = "#{@tag}.#{facility}.#{priority}"
 
       Engine.emit(tag, time, record)
+    rescue => e
+      $log.error "syslog failed to emit", :error => e.to_s, :error_class => e.class.to_s, :tag => tag, :record => Yajl.dump(record)
     end
 
     class UdpHandler < Coolio::IO

data/lib/fluent/supervisor.rb

@@ -18,11 +18,12 @@
 module Fluent
   class Supervisor
     class LoggerInitializer
-      def initialize(path, level, chuser, chgroup)
+      def initialize(path, level, chuser, chgroup, opts)
         @path = path
         @level = level
         @chuser = chuser
         @chgroup = chgroup
+        @opts = opts
       end
 
       def init
@@ -37,7 +38,7 @@ module Fluent
           @io = STDOUT
         end
 
-        $log = Fluent::Log.new(@io, @level)
+        $log = Fluent::Log.new(@io, @level, @opts)
 
         $log.enable_color(false) if @path
         $log.enable_debug if @level <= Fluent::Log::LEVEL_DEBUG
@@ -69,7 +70,8 @@ module Fluent
       @dry_run = opt[:dry_run]
       @suppress_config_dump = opt[:suppress_config_dump]
 
-      @log = LoggerInitializer.new(@log_path, @log_level, @chuser, @chgroup)
+      log_opts = {:suppress_repeated_stacktrace => opt[:suppress_repeated_stacktrace]}
+      @log = LoggerInitializer.new(@log_path, @log_level, @chuser, @chgroup, log_opts)
       @finished = false
       @main_pid = nil
     end

data/lib/fluent/version.rb

@@ -1,5 +1,5 @@
 module Fluent
 
-  VERSION = '0.10.40'
+  VERSION = '0.10.41'
 
 end

data/test/output.rb

@@ -0,0 +1,55 @@
+require 'helper'
+require 'fluent/test'
+require 'fluent/output'
+
+module FluentOutputTest
+  include Fluent
+
+  class BufferedOutputTest < ::Test::Unit::TestCase
+    include FluentOutputTest
+
+    def setup
+      Fluent::Test.setup
+    end
+
+    CONFIG = %[]
+
+    def create_driver(conf=CONFIG)
+      Fluent::Test::BufferedOutputTestDriver.new(Fluent::BufferedOutput) do
+        def write(chunk)
+          chunk.read
+        end
+      end.configure(conf)
+    end
+
+    def test_configure
+      # default
+      d = create_driver
+      assert_equal 'memory', d.instance.buffer_type
+      assert_equal 60, d.instance.flush_interval
+      assert_equal 17, d.instance.retry_limit
+      assert_equal 1.0, d.instance.retry_wait
+      assert_equal nil, d.instance.max_retry_wait
+      assert_equal 1.0, d.instance.retry_wait
+      assert_equal 1, d.instance.num_threads
+      assert_equal 1, d.instance.queued_chunk_flush_interval
+
+      # max_retry_wait
+      d = create_driver(CONFIG + %[max_retry_wait 4])
+      assert_equal 4, d.instance.max_retry_wait
+    end
+
+    def test_calc_retry_wait
+      # default
+      d = create_driver
+      d.instance.retry_limit.times { d.instance.instance_variable_get(:@error_history) << Engine.now }
+      wait = d.instance.retry_wait * (2 ** (d.instance.retry_limit - 1))
+      assert( d.instance.calc_retry_wait > wait - wait / 8.0 )
+
+      # max_retry_wait
+      d = create_driver(CONFIG + %[max_retry_wait 4])
+      d.instance.retry_limit.times { d.instance.instance_variable_get(:@error_history) << Engine.now }
+      assert_equal 4, d.instance.calc_retry_wait
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.10.40
+  version: 0.10.41
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-08 00:00:00.000000000 Z
+date: 2013-12-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -268,6 +268,7 @@ files:
 - test/helper.rb
 - test/match.rb
 - test/mixin.rb
+- test/output.rb
 - test/parser.rb
 - test/plugin/in_exec.rb
 - test/plugin/in_forward.rb
@@ -307,7 +308,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -2484756068667475409
+      hash: -602194787130919488
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23
@@ -320,6 +321,7 @@ test_files:
 - test/helper.rb
 - test/match.rb
 - test/mixin.rb
+- test/output.rb
 - test/parser.rb
 - test/plugin/in_exec.rb
 - test/plugin/in_forward.rb