fluentd 0.10.42 → 0.10.43

data/lib/fluent/plugin/in_http.rb

@@ -33,6 +33,7 @@ module Fluent
     config_param :body_size_limit, :size, :default => 32*1024*1024  # TODO default
     config_param :keepalive_timeout, :time, :default => 10   # TODO default
     config_param :backlog, :integer, :default => nil
+    config_param :add_http_headers, :bool, :default => false
 
     def configure(conf)
       super
@@ -70,14 +71,14 @@ module Fluent
     end
 
     def start
-      $log.debug "listening http on #{@bind}:#{@port}"
+      log.debug "listening http on #{@bind}:#{@port}"
       lsock = TCPServer.new(@bind, @port)
 
       detach_multi_process do
         super
         @km = KeepaliveManager.new(@keepalive_timeout)
         #@lsock = Coolio::TCPServer.new(@bind, @port, Handler, @km, method(:on_request), @body_size_limit)
-        @lsock = Coolio::TCPServer.new(lsock, nil, Handler, @km, method(:on_request), @body_size_limit)
+        @lsock = Coolio::TCPServer.new(lsock, nil, Handler, @km, method(:on_request), @body_size_limit, log)
         @lsock.listen(@backlog) unless @backlog.nil?
 
         @loop = Coolio::Loop.new
@@ -98,8 +99,8 @@ module Fluent
     def run
       @loop.run
     rescue
-      $log.error "unexpected error", :error=>$!.to_s
-      $log.error_backtrace
+      log.error "unexpected error", :error=>$!.to_s
+      log.error_backtrace
     end
 
     def on_request(path_info, params)
@@ -121,6 +122,14 @@ module Fluent
         if record.nil?
           return ["200 OK", {'Content-type'=>'text/plain'}, ""]
         end
+        
+        if @add_http_headers
+          params.each_pair { |k,v|
+            if k.start_with?("HTTP_")
+              record[k] = v
+            end
+          }
+        end
 
         time = params['time']
         time = time.to_i
@@ -143,13 +152,14 @@ module Fluent
     end
 
     class Handler < Coolio::Socket
-      def initialize(io, km, callback, body_size_limit)
+      def initialize(io, km, callback, body_size_limit, log)
         super(io)
         @km = km
         @callback = callback
         @body_size_limit = body_size_limit
         @content_type = ""
         @next_close = false
+        @log = log
 
         @idle = 0
         @km.add(self)
@@ -173,8 +183,8 @@ module Fluent
         @idle = 0
         @parser << data
       rescue
-        $log.warn "unexpected error", :error=>$!.to_s
-        $log.warn_backtrace
+        @log.warn "unexpected error", :error=>$!.to_s
+        @log.warn_backtrace
         close
       end
 

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -154,7 +154,7 @@ module Fluent
     end
 
     def start
-      $log.debug "listening monitoring http server on http://#{@bind}:#{@port}/api/plugins"
+      log.debug "listening monitoring http server on http://#{@bind}:#{@port}/api/plugins"
       @srv = WEBrick::HTTPServer.new({
           :BindAddress => @bind,
           :Port => @port,

data/lib/fluent/plugin/in_object_space.rb

@@ -28,8 +28,9 @@ module Fluent
     config_param :top, :integer, :default => 15
 
     class TimerWatcher < Coolio::TimerWatcher
-      def initialize(interval, repeat, &callback)
+      def initialize(interval, repeat, log, &callback)
         @callback = callback
+        @log = log
         super(interval, repeat)
       end
 
@@ -37,8 +38,8 @@ module Fluent
         @callback.call
       rescue
         # TODO log?
-        $log.error $!.to_s
-        $log.error_backtrace
+        @log.error $!.to_s
+        @log.error_backtrace
       end
     end
 
@@ -48,7 +49,7 @@ module Fluent
 
     def start
       @loop = Coolio::Loop.new
-      @timer = TimerWatcher.new(@emit_interval, true, &method(:on_timer))
+      @timer = TimerWatcher.new(@emit_interval, true, log, &method(:on_timer))
       @loop.attach(@timer)
       @thread = Thread.new(&method(:run))
     end
@@ -62,8 +63,8 @@ module Fluent
     def run
       @loop.run
     rescue
-      $log.error "unexpected error", :error=>$!.to_s
-      $log.error_backtrace
+      log.error "unexpected error", :error=>$!.to_s
+      log.error_backtrace
     end
 
     class Counter
@@ -110,7 +111,7 @@ module Fluent
 
       Engine.emit(@tag, now, record)
     rescue => e
-      $log.error "object space failed to emit", :error => e.to_s, :error_class => e.class.to_s, :tag => @tag, :record => Yajl.dump(record)
+      log.error "object space failed to emit", :error => e.to_s, :error_class => e.class.to_s, :tag => @tag, :record => Yajl.dump(record)
     end
   end
 end

data/lib/fluent/plugin/in_status.rb

@@ -27,8 +27,9 @@ module Fluent
     config_param :tag, :string
 
     class TimerWatcher < Coolio::TimerWatcher
-      def initialize(interval, repeat, &callback)
+      def initialize(interval, repeat, log, &callback)
         @callback = callback
+        @log = log
         super(interval, repeat)
       end
 
@@ -36,8 +37,8 @@ module Fluent
         @callback.call
       rescue
         # TODO log?
-        $log.error $!.to_s
-        $log.error_backtrace
+        @log.error $!.to_s
+        @log.error_backtrace
       end
     end
 
@@ -47,7 +48,7 @@ module Fluent
 
     def start
       @loop = Coolio::Loop.new
-      @timer = TimerWatcher.new(@emit_interval, true, &method(:on_timer))
+      @timer = TimerWatcher.new(@emit_interval, true, log, &method(:on_timer))
       @loop.attach(@timer)
       @thread = Thread.new(&method(:run))
     end
@@ -61,8 +62,8 @@ module Fluent
     def run
       @loop.run
     rescue
-      $log.error "unexpected error", :error=>$!.to_s
-      $log.error_backtrace
+      log.error "unexpected error", :error=>$!.to_s
+      log.error_backtrace
     end
 
     def on_timer

data/lib/fluent/plugin/in_stream.rb

@@ -45,8 +45,8 @@ module Fluent
     def run
       @loop.run
     rescue
-      $log.error "unexpected error", :error=>$!.to_s
-      $log.error_backtrace
+      log.error "unexpected error", :error=>$!.to_s
+      log.error_backtrace
     end
 
     protected
@@ -104,14 +104,15 @@ module Fluent
     end
 
     class Handler < Coolio::Socket
-      def initialize(io, on_message)
+      def initialize(io, log, on_message)
         super(io)
         if io.is_a?(TCPSocket)
           opt = [1, @timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
           io.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
         end
-        $log.trace { "accepted fluent socket object_id=#{self.object_id}" }
         @on_message = on_message
+        @log = log
+        @log.trace { "accepted fluent socket object_id=#{self.object_id}" }
       end
 
       def on_connect
@@ -137,21 +138,21 @@ module Fluent
       def on_read_json(data)
         @y << data
       rescue
-        $log.error "unexpected error", :error=>$!.to_s
-        $log.error_backtrace
+        @log.error "unexpected error", :error=>$!.to_s
+        @log.error_backtrace
         close
       end
 
       def on_read_msgpack(data)
         @u.feed_each(data, &@on_message)
       rescue
-        $log.error "unexpected error", :error=>$!.to_s
-        $log.error_backtrace
+        @log.error "unexpected error", :error=>$!.to_s
+        @log.error_backtrace
         close
       end
 
       def on_close
-        $log.trace { "closed fluent socket object_id=#{self.object_id}" }
+        @log.trace { "closed fluent socket object_id=#{self.object_id}" }
       end
     end
   end
@@ -170,7 +171,7 @@ module Fluent
   #  end
   #
   #  def listen
-  #    $log.debug "listening fluent socket on #{@bind}:#{@port}"
+  #    log.debug "listening fluent socket on #{@bind}:#{@port}"
   #    Coolio::TCPServer.new(@bind, @port, Handler, method(:on_message))
   #  end
   #end
@@ -192,7 +193,7 @@ module Fluent
 
     def configure(conf)
       super
-      #$log.warn "'unix' input is obsoleted and will be removed. Use 'forward' instead."
+      #log.warn "'unix' input is obsoleted and will be removed. Use 'forward' instead."
     end
 
     def listen
@@ -200,8 +201,8 @@ module Fluent
         File.unlink(@path)
       end
       FileUtils.mkdir_p File.dirname(@path)
-      $log.debug "listening fluent socket on #{@path}"
-      s = Coolio::UNIXServer.new(@path, Handler, method(:on_message))
+      log.debug "listening fluent socket on #{@path}"
+      s = Coolio::UNIXServer.new(@path, Handler, log, method(:on_message))
       s.listen(@backlog) unless @backlog.nil?
       s
     end

data/lib/fluent/plugin/in_syslog.rb

@@ -117,15 +117,15 @@ module Fluent
     def run
       @loop.run
     rescue
-      $log.error "unexpected error", :error=>$!.to_s
-      $log.error_backtrace
+      log.error "unexpected error", :error=>$!.to_s
+      log.error_backtrace
     end
 
     protected
     def receive_data_parser(data)
       m = SYSLOG_REGEXP.match(data)
       unless m
-        $log.debug "invalid syslog message: #{data.dump}"
+        log.debug "invalid syslog message: #{data.dump}"
         return
       end
       pri = m[1].to_i
@@ -133,20 +133,21 @@ module Fluent
 
       time, record = @parser.parse(text)
       unless time && record
+        log.warn "pattern not match: #{text.inspect}"
         return
       end
 
       emit(pri, time, record)
 
     rescue
-      $log.warn data.dump, :error=>$!.to_s
-      $log.debug_backtrace
+      log.warn data.dump, :error=>$!.to_s
+      log.debug_backtrace
     end
 
     def receive_data(data)
       m = SYSLOG_ALL_REGEXP.match(data)
       unless m
-        $log.debug "invalid syslog message", :data=>data
+        log.debug "invalid syslog message", :data=>data
         return
       end
 
@@ -172,20 +173,20 @@ module Fluent
       emit(pri, time, record)
 
     rescue
-      $log.warn data.dump, :error=>$!.to_s
-      $log.debug_backtrace
+      log.warn data.dump, :error=>$!.to_s
+      log.debug_backtrace
     end
 
     private
 
     def listen(callback)
-      $log.debug "listening syslog socket on #{@bind}:#{@port} with #{@protocol_type}"
+      log.debug "listening syslog socket on #{@bind}:#{@port} with #{@protocol_type}"
       if @protocol_type == :udp
         @usock = SocketUtil.create_udp_socket(@bind)
         @usock.bind(@bind, @port)
         UdpHandler.new(@usock, callback)
       else
-        Coolio::TCPServer.new(@bind, @port, TcpHandler, callback)
+        Coolio::TCPServer.new(@bind, @port, TcpHandler, log, callback)
       end
     end
 
@@ -197,7 +198,7 @@ module Fluent
 
       Engine.emit(tag, time, record)
     rescue => e
-      $log.error "syslog failed to emit", :error => e.to_s, :error_class => e.class.to_s, :tag => tag, :record => Yajl.dump(record)
+      log.error "syslog failed to emit", :error => e.to_s, :error_class => e.class.to_s, :tag => tag, :record => Yajl.dump(record)
     end
 
     class UdpHandler < Coolio::IO
@@ -219,14 +220,15 @@ module Fluent
     end
 
     class TcpHandler < Coolio::Socket
-      def initialize(io, on_message)
+      def initialize(io, log, on_message)
         super(io)
         if io.is_a?(TCPSocket)
           opt = [1, @timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
           io.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
         end
-        $log.trace { "accepted fluent socket object_id=#{self.object_id}" }
         @on_message = on_message
+        @log = log
+        @log.trace { "accepted fluent socket object_id=#{self.object_id}" }
         @buffer = "".force_encoding('ASCII-8BIT')
       end
 
@@ -245,12 +247,12 @@ module Fluent
         end
         @buffer.slice!(0, pos) if pos > 0
       rescue => e
-        $log.error "syslog error", :error => e, :error_class => e.class
+        @log.error "syslog error", :error => e, :error_class => e.class
         close
       end
 
       def on_close
-        $log.trace { "closed fluent socket object_id=#{self.object_id}" }
+        @log.trace { "closed fluent socket object_id=#{self.object_id}" }
       end
     end
   end

data/lib/fluent/plugin/in_tail.rb

@@ -62,7 +62,9 @@ module Fluent
       @loop = Coolio::Loop.new
       @tails = @paths.map {|path|
         pe = @pf ? @pf[path] : MemoryPositionEntry.new
-        TailWatcher.new(path, @rotate_wait, pe, &method(:receive_lines))
+        tw = TailWatcher.new(path, @rotate_wait, pe, &method(:receive_lines))
+        tw.log = log
+        tw
       }
       @tails.each {|tail|
         tail.attach(@loop)
@@ -82,8 +84,8 @@ module Fluent
     def run
       @loop.run
     rescue
-      $log.error "unexpected error", :error=>$!.to_s
-      $log.error_backtrace
+      log.error "unexpected error", :error=>$!.to_s
+      log.error_backtrace
     end
 
     def receive_lines(lines)
@@ -94,10 +96,12 @@ module Fluent
           time, record = parse_line(line)
           if time && record
             es.add(time, record)
+          else
+            log.warn "pattern not match: #{line.inspect}"
           end
         rescue
-          $log.warn line.dump, :error=>$!.to_s
-          $log.debug_backtrace
+          log.warn line.dump, :error=>$!.to_s
+          log.debug_backtrace
         end
       }
 
@@ -128,6 +132,19 @@ module Fluent
 
         @rotate_handler = RotateHandler.new(path, &method(:on_rotate))
         @io_handler = nil
+        @log = $log
+      end
+
+      # We use accessor approach to assign each logger, not passing log object at initialization,
+      # because several plugins depend on these internal classes.
+      # This approach avoids breaking plugins with new log_level option.
+      attr_accessor :log
+
+      def log=(logger)
+        @log = logger
+        @timer_trigger.log = logger
+        @stat_trigger.log = logger
+        @rotate_handler.log = logger
       end
 
       def attach(loop)
@@ -173,7 +190,7 @@ module Fluent
               pos = io.pos
             end
             @pe.update(inode, pos)
-            io_handler = IOHandler.new(io, @pe, &@receive_lines)
+            io_handler = IOHandler.new(io, @pe, log, &@receive_lines)
           else
             io_handler = NullIOHandler.new
           end
@@ -212,7 +229,7 @@ module Fluent
             end
             io.seek(pos)
 
-            @io_handler = IOHandler.new(io, @pe, &@receive_lines)
+            @io_handler = IOHandler.new(io, @pe, log, &@receive_lines)
           else
             @io_handler = NullIOHandler.new
           end
@@ -223,11 +240,11 @@ module Fluent
           end
           last_io = @rotate_queue.empty? ? @io_handler.io : @rotate_queue.last.io
           if last_io == nil
-            $log.info "detected rotation of #{@path}"
+            log.info "detected rotation of #{@path}"
             # rotate imeediately if previous file is nil
             wait = 0
           else
-            $log.info "detected rotation of #{@path}; waiting #{@rotate_wait} seconds"
+            log.info "detected rotation of #{@path}; waiting #{@rotate_wait} seconds"
             wait = @rotate_wait
             wait -= @rotate_queue.first.wait unless @rotate_queue.empty?
           end
@@ -238,30 +255,36 @@ module Fluent
       class TimerWatcher < Coolio::TimerWatcher
         def initialize(interval, repeat, &callback)
           @callback = callback
+          @log = $log
           super(interval, repeat)
         end
 
+        attr_accessor :log
+
         def on_timer
           @callback.call
         rescue
           # TODO log?
-          $log.error $!.to_s
-          $log.error_backtrace
+          @log.error $!.to_s
+          @log.error_backtrace
         end
       end
 
       class StatWatcher < Coolio::StatWatcher
         def initialize(path, &callback)
           @callback = callback
+          @log = $log
           super(path)
         end
 
+        attr_accessor :log
+
         def on_change(prev, cur)
           @callback.call
         rescue
           # TODO log?
-          $log.error $!.to_s
-          $log.error_backtrace
+          @log.error $!.to_s
+          @log.error_backtrace
         end
       end
 
@@ -285,8 +308,9 @@ module Fluent
       MAX_LINES_AT_ONCE = 1000
 
       class IOHandler
-        def initialize(io, pe, &receive_lines)
-          $log.info "following tail of #{io.path}"
+        def initialize(io, pe, log, &receive_lines)
+          @log = log
+          @log.info "following tail of #{io.path}"
           @io = io
           @pe = pe
           @receive_lines = receive_lines
@@ -328,8 +352,8 @@ module Fluent
           end while read_more
 
         rescue
-          $log.error $!.to_s
-          $log.error_backtrace
+          @log.error $!.to_s
+          @log.error_backtrace
           close
         end
 
@@ -358,8 +382,11 @@ module Fluent
           @inode = nil
           @fsize = -1  # first
           @on_rotate = on_rotate
+          @log = $log
         end
 
+        attr_accessor :log
+
         def on_notify
           begin
             io = File.open(@path)
@@ -386,8 +413,8 @@ module Fluent
           end
 
         rescue
-          $log.error $!.to_s
-          $log.error_backtrace
+          @log.error $!.to_s
+          @log.error_backtrace
         end
       end
     end