fluentd 0.10.42 → 0.10.43

data/ChangeLog

@@ -1,3 +1,12 @@
+Release 0.10.43 - 2013/02/03
+
+* Input/Output: Add 'log_level' option to plugins
+* parser/in_tail: Fix RegexpParser forget to call configure method
+* parser/in_tail: Improve type conversion performance
+* in_http: Add add_http_headers option to add HTTP headers to record
+* Buffer: Improve submit_buffer throughput using round-robin writer selection
+* add sigdump gem to dump useful information via SIGCONT
+
 Release 0.10.42 - 2013/12/25
 
 * parser/in_tail: Add 'types' and related parameters to convert record value type

data/README.rdoc

@@ -1,6 +1,6 @@
 = Fluentd
 
-{<img src="https://travis-ci.org/fluent/fluentd.png" />}[https://travis-ci.org/fluent/fluentd]
+{<img src="https://travis-ci.org/fluent/fluentd.png" />}[https://travis-ci.org/fluent/fluentd] {<img src="https://codeclimate.com/github/fluent/fluentd.png " />}[https://codeclimate.com/github/fluent/fluentd]
 
 Fluentd is an event collector system. It is a generalized version of syslogd, which handles JSON objects for its log messages.
 
@@ -64,3 +64,6 @@ License:: Apache License, Version 2.0
 == Contributors:
 
 Patches contributed by {those people}[https://github.com/fluent/fluentd/contributors].
+
+{<img src="https://ga-beacon.appspot.com/UA-24890265-6/fluent/fluentd" />}[https://github.com/fluent/fluentd]
+

data/fluentd.gemspec

@@ -23,6 +23,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency(%q<yajl-ruby>, ["~> 1.0"])
   gem.add_runtime_dependency(%q<cool.io>, ["~> 1.1.1"])
   gem.add_runtime_dependency(%q<http_parser.rb>, ["~> 0.5.1"])
+  gem.add_runtime_dependency(%q<sigdump>, ["~> 0.2.2"])
 
   gem.add_development_dependency(%q<rake>, [">= 0.9.2"])
   gem.add_development_dependency(%q<parallel_tests>, [">= 0.15.3"])

data/lib/fluent/input.rb

@@ -19,6 +19,7 @@ module Fluent
   class Input
     include Configurable
     include PluginId
+    include PluginLoggerMixin
 
     def initialize
       super
@@ -35,4 +36,3 @@ module Fluent
     end
   end
 end
-

data/lib/fluent/load.rb

@@ -9,6 +9,7 @@ require 'json'
 require 'yajl'
 require 'uri'
 require 'msgpack'
+require 'sigdump/setup'
 # I hate global variable but we suffer pain now for the sake of future.
 # We will switch to MessagePack 0.5 and deprecate 0.4.
 $use_msgpack_5 = defined?(MessagePack::Packer) ? true : false

data/lib/fluent/log.rb

@@ -15,6 +15,9 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
+require 'forwardable'
+
 module Fluent
   class Log
     module TTYColor
@@ -47,6 +50,7 @@ module Fluent
       @self_event = false
       @tag = 'fluent'
       @time_format = '%Y-%m-%d %H:%M:%S %z '
+      @depth_offset = 1
       enable_color out.tty?
       # TODO: This variable name is unclear so we should change to better name.
       @threads_exclude_events = []
@@ -71,6 +75,10 @@ module Fluent
       self
     end
 
+    def enable_color?
+      !@color_reset.empty?
+    end
+
     def enable_color(b=true)
       if b
         @color_trace = TTYColor::BLUE
@@ -107,7 +115,7 @@ module Fluent
       return if @level > LEVEL_TRACE
       args << block.call if block
       time, msg = event(:trace, args)
-      puts [@color_trace, caller_line(time, 1, LEVEL_TRACE), msg, @color_reset].join
+      puts [@color_trace, caller_line(time, @depth_offset, LEVEL_TRACE), msg, @color_reset].join
     rescue
       # logger should not raise an exception. This rescue prevents unexpected behaviour.
     end
@@ -126,7 +134,7 @@ module Fluent
       return if @level > LEVEL_DEBUG
       args << block.call if block
       time, msg = event(:debug, args)
-      puts [@color_debug, caller_line(time, 1, LEVEL_DEBUG), msg, @color_reset].join
+      puts [@color_debug, caller_line(time, @depth_offset, LEVEL_DEBUG), msg, @color_reset].join
     rescue
     end
     alias DEBUG debug
@@ -144,7 +152,7 @@ module Fluent
       return if @level > LEVEL_INFO
       args << block.call if block
       time, msg = event(:info, args)
-      puts [@color_info, caller_line(time, 1, LEVEL_INFO), msg, @color_reset].join
+      puts [@color_info, caller_line(time, @depth_offset, LEVEL_INFO), msg, @color_reset].join
     rescue
     end
     alias INFO info
@@ -162,7 +170,7 @@ module Fluent
       return if @level > LEVEL_WARN
       args << block.call if block
       time, msg = event(:warn, args)
-      puts [@color_warn, caller_line(time, 1, LEVEL_WARN), msg, @color_reset].join
+      puts [@color_warn, caller_line(time, @depth_offset, LEVEL_WARN), msg, @color_reset].join
     rescue
     end
     alias WARN warn
@@ -180,7 +188,7 @@ module Fluent
       return if @level > LEVEL_ERROR
       args << block.call if block
       time, msg = event(:error, args)
-      puts [@color_error, caller_line(time, 1, LEVEL_ERROR), msg, @color_reset].join
+      puts [@color_error, caller_line(time, @depth_offset, LEVEL_ERROR), msg, @color_reset].join
     rescue
     end
     alias ERROR error
@@ -198,7 +206,7 @@ module Fluent
       return if @level > LEVEL_FATAL
       args << block.call if block
       time, msg = event(:fatal, args)
-      puts [@color_fatal, caller_line(time, 1, LEVEL_FATAL), msg, @color_reset].join
+      puts [@color_fatal, caller_line(time, @depth_offset, LEVEL_FATAL), msg, @color_reset].join
     rescue
     end
     alias FATAL fatal
@@ -230,11 +238,12 @@ module Fluent
       return if @level > level
 
       time = Time.now
+      line = caller_line(time, 5, level)
       if @suppress_repeated_stacktrace && (Thread.current[:last_repeated_stacktrace] == backtrace)
-        puts ["  ", caller_line(time, 5, level), 'suppressed same stacktrace'].join
+        puts ["  ", line, 'suppressed same stacktrace'].join
       else
         backtrace.each { |msg|
-          puts ["  ", caller_line(time, 5, level), msg].join
+          puts ["  ", line, msg].join
         }
         Thread.current[:last_repeated_stacktrace] = backtrace if @suppress_repeated_stacktrace
       end
@@ -286,5 +295,71 @@ module Fluent
       return log_msg
     end
   end
-end
 
+
+  # PluginLogger has own log level separated from global $log object.
+  # This class enables log_level option in each plugin.
+  #
+  # PluginLogger has same functionality as Log but some methods are forwarded to internal logger
+  # for keeping logging action consistency in the process, e.g. color, tag, event, etc.
+  class PluginLogger < Log
+    def initialize(logger)
+      @logger = logger
+      @level = @logger.level
+      @depth_offset = 2
+      @suppress_repeated_stacktrace = logger.instance_variable_get(:@suppress_repeated_stacktrace)
+
+      enable_color @logger.enable_color?
+    end
+
+    def level=(log_level_str)
+      @level = case log_level_str.downcase
+               when "trace" then LEVEL_TRACE
+               when "debug" then LEVEL_DEBUG
+               when "info"  then LEVEL_INFO
+               when "warn"  then LEVEL_WARN
+               when "error" then LEVEL_ERROR
+               when "fatal" then LEVEL_FATAL
+               else raise "Unknown log level: level = #{log_level_str}"
+               end
+    end
+
+    alias orig_enable_color enable_color
+
+    def enable_color(b = true)
+      orig_enable_color b
+      @logger.enable_color b
+    end
+
+    extend Forwardable
+    def_delegators '@logger', :enable_color?, :enable_debug, :enable_event,
+      :disable_events, :tag, :tag=, :time_format, :time_format=,
+      :event, :caller_line, :puts, :write, :flush
+  end
+
+
+  module PluginLoggerMixin
+    def self.included(klass)
+      klass.instance_eval {
+        config_param :log_level, :string, :default => nil
+      }
+    end
+
+    def initialize
+      super
+
+      @log = $log # Use $log object directly by default
+    end
+
+    attr_accessor :log
+
+    def configure(conf)
+      super
+
+      if @log_level
+        @log = PluginLogger.new($log)
+        @log.level = @log_level
+      end
+    end
+  end
+end

data/lib/fluent/output.rb

@@ -59,6 +59,7 @@ module Fluent
   class Output
     include Configurable
     include PluginId
+    include PluginLoggerMixin
 
     def initialize
       super
@@ -223,6 +224,8 @@ module Fluent
       @buffer.start
       @secondary.start if @secondary
       @writers.each {|writer| writer.start }
+      @writer_current_position = 0
+      @writers_size = @writers.size
     end
 
     def shutdown
@@ -240,8 +243,9 @@ module Fluent
     end
 
     def submit_flush
-      # TODO roundrobin?
-      @writers.first.submit_flush
+      # Without locks: it is rough but enough to select "next" writer selection
+      @writer_current_position = (@writer_current_position + 1) % @writers_size
+      @writers[@writer_current_position].submit_flush
     end
 
     def format_stream(tag, es)

data/lib/fluent/parser.rb

@@ -137,10 +137,14 @@ module Fluent
         @mutex = Mutex.new
       end
 
+      def configure(conf)
+        super
+        @time_parser = TimeParser.new(@time_format)
+      end
+
       def call(text)
         m = @regexp.match(text)
         unless m
-          $log.warn "pattern not match: #{text.inspect}"
           return nil, nil
         end
 
@@ -198,7 +202,6 @@ module Fluent
 
         return time, record
       rescue Yajl::ParseError
-        $log.warn "pattern not match: #{text.inspect}: #{$!}"
         return nil, nil
       end
     end
@@ -246,8 +249,10 @@ module Fluent
       private
 
       def convert_field_type!(record)
-        record.each { |key, value|
-          record[key] = convert_type(key, value)
+        @type_converters.each_key { |key|
+          if value = record[key]
+            record[key] = convert_type(key, value)
+          end
         }
       end
     end
@@ -320,7 +325,6 @@ module Fluent
       def call(text)
         m = REGEXP.match(text)
         unless m
-          $log.warn "pattern not match: #{text.inspect}"
           return nil, nil
         end
 

data/lib/fluent/plugin/in_debug_agent.rb

@@ -41,7 +41,7 @@ module Fluent
       else
         uri = "druby://#{@bind}:#{@port}"
       end
-      $log.info "listening dRuby", :uri => uri, :object => @object
+      log.info "listening dRuby", :uri => uri, :object => @object
       obj = eval(@object)
       @server = DRb::DRbServer.new(uri, obj)
     end

data/lib/fluent/plugin/in_exec.rb

@@ -124,8 +124,8 @@ module Fluent
           @parser.call(io)
           Process.waitpid(io.pid)
         rescue
-          $log.error "exec failed to run or shutdown child process", :error => $!.to_s, :error_class => $!.class.to_s
-          $log.warn_backtrace $!.backtrace
+          log.error "exec failed to run or shutdown child process", :error => $!.to_s, :error_class => $!.class.to_s
+          log.warn_backtrace $!.backtrace
         end
       end
     end
@@ -147,7 +147,7 @@ module Fluent
 
       Engine.emit(tag, time, record)
     rescue => e
-      $log.error "exec failed to emit", :error => e.to_s, :error_class => e.class.to_s, :tag => tag, :record => Yajl.dump(record)
+      log.error "exec failed to emit", :error => e.to_s, :error_class => e.class.to_s, :tag => tag, :record => Yajl.dump(record)
     end
   end
 end

data/lib/fluent/plugin/in_forward.rb

@@ -61,8 +61,8 @@ module Fluent
     end
 
     def listen
-      $log.info "listening fluent socket on #{@bind}:#{@port}"
-      s = Coolio::TCPServer.new(@bind, @port, Handler, method(:on_message))
+      log.info "listening fluent socket on #{@bind}:#{@port}"
+      s = Coolio::TCPServer.new(@bind, @port, Handler, log, method(:on_message))
       s.listen(@backlog) unless @backlog.nil?
       s
     end
@@ -73,15 +73,15 @@ module Fluent
     #    File.unlink(@path)
     #  end
     #  FileUtils.mkdir_p File.dirname(@path)
-    #  $log.debug "listening fluent socket on #{@path}"
+    #  log.debug "listening fluent socket on #{@path}"
     #  Coolio::UNIXServer.new(@path, Handler, method(:on_message))
     #end
 
     def run
       @loop.run
     rescue => e
-      $log.error "unexpected error", :error => e, :error_class => e.class
-      $log.error_backtrace
+      log.error "unexpected error", :error => e, :error_class => e.class
+      log.error_backtrace
     end
 
     protected
@@ -143,14 +143,15 @@ module Fluent
     end
 
     class Handler < Coolio::Socket
-      def initialize(io, on_message)
+      def initialize(io, log, on_message)
         super(io)
         if io.is_a?(TCPSocket)
           opt = [1, @timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
           io.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
         end
-        $log.trace { "accepted fluent socket object_id=#{self.object_id}" }
         @on_message = on_message
+        @log = log
+        @log.trace { "accepted fluent socket object_id=#{self.object_id}" }
       end
 
       def on_connect
@@ -176,21 +177,21 @@ module Fluent
       def on_read_json(data)
         @y << data
       rescue => e
-        $log.error "forward error", :error => e, :error_class => e.class
-        $log.error_backtrace
+        @log.error "forward error", :error => e, :error_class => e.class
+        @log.error_backtrace
         close
       end
 
       def on_read_msgpack(data)
         @u.feed_each(data, &@on_message)
       rescue => e
-        $log.error "forward error", :error => e, :error_class => e.class
-        $log.error_backtrace
+        @log.error "forward error", :error => e, :error_class => e.class
+        @log.error_backtrace
         close
       end
 
       def on_close
-        $log.trace { "closed fluent socket object_id=#{self.object_id}" }
+        @log.trace { "closed fluent socket object_id=#{self.object_id}" }
       end
     end
 
@@ -216,7 +217,7 @@ module Fluent
     end
 
     def on_heartbeat_request(host, port, msg)
-      #$log.trace "heartbeat request from #{host}:#{port}"
+      #log.trace "heartbeat request from #{host}:#{port}"
       begin
         @usock.send "\0", 0, host, port
       rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR

data/lib/fluent/plugin/in_gc_stat.rb

@@ -27,8 +27,9 @@ module Fluent
     config_param :tag, :string
 
     class TimerWatcher < Coolio::TimerWatcher
-      def initialize(interval, repeat, &callback)
+      def initialize(interval, repeat, log, &callback)
         @callback = callback
+        @log = log
         super(interval, repeat)
       end
 
@@ -36,8 +37,8 @@ module Fluent
         @callback.call
       rescue
         # TODO log?
-        $log.error $!.to_s
-        $log.error_backtrace
+        @log.error $!.to_s
+        @log.error_backtrace
       end
     end
 
@@ -47,7 +48,7 @@ module Fluent
 
     def start
       @loop = Coolio::Loop.new
-      @timer = TimerWatcher.new(@emit_interval, true, &method(:on_timer))
+      @timer = TimerWatcher.new(@emit_interval, true, log, &method(:on_timer))
       @loop.attach(@timer)
       @thread = Thread.new(&method(:run))
     end
@@ -61,8 +62,8 @@ module Fluent
     def run
       @loop.run
     rescue
-      $log.error "unexpected error", :error=>$!.to_s
-      $log.error_backtrace
+      log.error "unexpected error", :error=>$!.to_s
+      log.error_backtrace
     end
 
     def on_timer