fluentd 0.10.50 → 0.10.51

data/lib/fluent/plugin/in_stream.rb

@@ -160,34 +160,6 @@ module Fluent
     end
   end
 
-
-  # obsolete
-  # ForwardInput is backward compatible with TcpInput
-  #class TcpInput < StreamInput
-  #  Plugin.register_input('tcp', self)
-  #
-  #  config_param :port, :integer, :default => DEFAULT_LISTEN_PORT
-  #  config_param :bind, :string, :default => '0.0.0.0'
-  #
-  #  def configure(conf)
-  #    super
-  #  end
-  #
-  #  def listen
-  #    log.debug "listening fluent socket on #{@bind}:#{@port}"
-  #    Coolio::TCPServer.new(@bind, @port, Handler, method(:on_message))
-  #  end
-  #end
-  class TcpInput < ForwardInput
-    Plugin.register_input('tcp', self)
-
-    def initialize
-      super
-      $log.warn "'tcp' input is obsoleted and will be removed soon. Use 'forward' instead."
-    end
-  end
-
-
   class UnixInput < StreamInput
     Plugin.register_input('unix', self)
 

data/lib/fluent/plugin/in_syslog.rb

@@ -20,8 +20,6 @@ module Fluent
     Plugin.register_input('syslog', self)
 
     SYSLOG_REGEXP = /^\<([0-9]+)\>(.*)/
-    SYSLOG_ALL_REGEXP = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?[^\:]*\: *(?<message>.*)$/
-    TIME_FORMAT = "%b %d %H:%M:%S"
 
     FACILITY_MAP = {
       0   => 'kern',
@@ -88,17 +86,19 @@ module Fluent
       if parser.configure(conf, false)
         @parser = parser
       else
-        @parser = nil
-        @time_parser = TextParser::TimeParser.new(TIME_FORMAT)
+        conf['with_priority'] = true
+        @parser = TextParser::SyslogParser.new
+        @parser.configure(conf)
+        @use_default = true
       end
     end
 
     def start
-      if @parser
-        callback = method(:receive_data_parser)
-      else
-        callback = method(:receive_data)
-      end
+      callback = if @use_default
+                   method(:receive_data)
+                 else
+                   method(:receive_data_parser)
+                 end
 
       @loop = Coolio::Loop.new
       @handler = listen(callback)
@@ -122,10 +122,10 @@ module Fluent
     end
 
     protected
-    def receive_data_parser(data)
+    def receive_data_parser(data, addr)
       m = SYSLOG_REGEXP.match(data)
       unless m
-        log.debug "invalid syslog message: #{data.dump}"
+        log.warn "invalid syslog message: #{data.dump}"
         return
       end
       pri = m[1].to_i
@@ -139,42 +139,24 @@ module Fluent
 
         emit(pri, time, record)
       }
-    rescue
-      log.warn data.dump, :error=>$!.to_s
-      log.debug_backtrace
+    rescue => e
+      log.error data.dump, :error => e.to_s
+      log.error_backtrace
     end
 
-    def receive_data(data)
-      m = SYSLOG_ALL_REGEXP.match(data)
-      unless m
-        log.debug "invalid syslog message", :data=>data
-        return
-      end
-
-      pri = nil
-      time = nil
-      record = {}
-
-      m.names.each {|name|
-        if value = m[name]
-          case name
-          when "pri"
-            pri = value.to_i
-          when "time"
-            time = @time_parser.parse(value.gsub(/ +/, ' '))
-          else
-            record[name] = value
-          end
+    def receive_data(data, addr)
+      @parser.call(data) { |time, record|
+        unless time && record
+          log.warn "invalid syslog message", :data => data
+          return
         end
-      }
-
-      time ||= Engine.now
-
-      emit(pri, time, record)
 
-    rescue
-      log.warn data.dump, :error=>$!.to_s
-      log.debug_backtrace
+        pri = record.delete('pri')
+        emit(pri, time, record)
+      }
+    rescue => e
+      log.error data.dump, :error => e.to_s
+      log.error_backtrace
     end
 
     private
@@ -184,9 +166,10 @@ module Fluent
       if @protocol_type == :udp
         @usock = SocketUtil.create_udp_socket(@bind)
         @usock.bind(@bind, @port)
-        UdpHandler.new(@usock, callback)
+        SocketUtil::UdpHandler.new(@usock, log, 2048, callback)
       else
-        Coolio::TCPServer.new(@bind, @port, TcpHandler, log, callback)
+        # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
+        Coolio::TCPServer.new(@bind, @port, SocketUtil::TcpHandler, log, "\n", callback)
       end
     end
 
@@ -200,60 +183,5 @@ module Fluent
     rescue => e
       log.error "syslog failed to emit", :error => e.to_s, :error_class => e.class.to_s, :tag => tag, :record => Yajl.dump(record)
     end
-
-    class UdpHandler < Coolio::IO
-      def initialize(io, callback)
-        super(io)
-        @io = io
-        @callback = callback
-      end
-
-      def on_readable
-        msg, addr = @io.recvfrom_nonblock(2048)
-        #host = addr[3]
-        #port = addr[1]
-        #@callback.call(host, port, msg)
-        @callback.call(msg)
-      rescue
-        # TODO log?
-      end
-    end
-
-    class TcpHandler < Coolio::Socket
-      def initialize(io, log, on_message)
-        super(io)
-        if io.is_a?(TCPSocket)
-          opt = [1, @timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
-          io.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
-        end
-        @on_message = on_message
-        @log = log
-        @log.trace { "accepted fluent socket object_id=#{self.object_id}" }
-        @buffer = "".force_encoding('ASCII-8BIT')
-      end
-
-      def on_connect
-      end
-
-      def on_read(data)
-        @buffer << data
-        pos = 0
-
-        # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
-        while i = @buffer.index("\n", pos)
-          msg = @buffer[pos..i]
-          @on_message.call(msg)
-          pos = i + 1
-        end
-        @buffer.slice!(0, pos) if pos > 0
-      rescue => e
-        @log.error "syslog error", :error => e, :error_class => e.class
-        close
-      end
-
-      def on_close
-        @log.trace { "closed fluent socket object_id=#{self.object_id}" }
-      end
-    end
   end
 end

data/lib/fluent/plugin/in_tail.rb

@@ -338,7 +338,7 @@ module Fluent
       end
 
       def on_notify
-        @rotate_handler.on_notify
+        @rotate_handler.on_notify if @rotate_handler
         return unless @io_handler
         @io_handler.on_notify
       end
@@ -396,12 +396,13 @@ module Fluent
               @pe.update(inode, io.pos)
               io_handler = IOHandler.new(io, @pe, @log, &method(:wrap_receive_lines))
               @io_handler = io_handler
-            else
+            else # file is rotated and new file found
               @update_watcher.call(@path, swap_state(@pe))
             end
-          else
-            @io_handler.close
-            @io_handler = NullIOHandler.new
+          else # file is rotated and new file not found
+            # Clear RotateHandler to avoid duplicated file watch in same path.
+            @rotate_handler = nil
+            @update_watcher.call(@path, swap_state(@pe))
           end
         end
 

data/lib/fluent/plugin/in_tcp.rb

@@ -0,0 +1,15 @@
+require 'fluent/plugin/socket_util'
+
+module Fluent
+  class TcpInput < SocketUtil::BaseInput
+    Plugin.register_input('tcp', self)
+
+    config_set_default :port, 5170
+    config_param :delimiter, :string, :default => "\n" # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
+
+    def listen(callback)
+      log.debug "listening tcp socket on #{@bind}:#{@port}"
+      Coolio::TCPServer.new(@bind, @port, SocketUtil::TcpHandler, log, @delimiter, callback)
+    end
+  end
+end

data/lib/fluent/plugin/in_udp.rb

@@ -0,0 +1,17 @@
+require 'fluent/plugin/socket_util'
+
+module Fluent
+  class UdpInput < SocketUtil::BaseInput
+    Plugin.register_input('udp', self)
+
+    config_set_default :port, 5160
+    config_param :body_size_limit, :size, :default => 4096
+
+    def listen(callback)
+      log.debug "listening udp socket on #{@bind}:#{@port}"
+      @usock = SocketUtil.create_udp_socket(@bind)
+      @usock.bind(@bind, @port)
+      SocketUtil::UdpHandler.new(@usock, log, @body_size_limit, callback)
+    end
+  end
+end

data/lib/fluent/plugin/socket_util.rb

@@ -1,3 +1,5 @@
+require 'cool.io'
+
 module Fluent
   module SocketUtil
     def create_udp_socket(host)
@@ -10,5 +12,122 @@ module Fluent
       end
     end
     module_function :create_udp_socket
+
+    class UdpHandler < Coolio::IO
+      def initialize(io, log, body_size_limit, callback)
+        super(io)
+        @io = io
+        @log = log
+        @body_size_limit = body_size_limit
+        @callback = callback
+      end
+
+      def on_readable
+        msg, addr = @io.recvfrom_nonblock(@body_size_limit)
+        msg.chomp!
+        @callback.call(msg, addr)
+      rescue => e
+        @log.error "unexpected error", :error => e, :error_class => e.class
+      end
+    end
+
+    class TcpHandler < Coolio::Socket
+      PEERADDR_FAILED = ["?", "?", "name resolusion failed", "?"]
+
+      def initialize(io, log, delimiter, callback)
+        super(io)
+        if io.is_a?(TCPSocket)
+          @addr = (io.peeraddr rescue PEERADDR_FAILED)
+
+          opt = [1, @timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
+          io.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
+        end
+        @delimiter = delimiter
+        @callback = callback
+        @log = log
+        @log.trace { "accepted fluent socket object_id=#{self.object_id}" }
+        @buffer = "".force_encoding('ASCII-8BIT')
+      end
+
+      def on_connect
+      end
+
+      def on_read(data)
+        @buffer << data
+        pos = 0
+
+        while i = @buffer.index(@delimiter, pos)
+          msg = @buffer[pos...i]
+          @callback.call(msg, @addr)
+          pos = i + @delimiter.length
+        end
+        @buffer.slice!(0, pos) if pos > 0
+      rescue => e
+        @log.error "unexpected error", :error => e, :error_class => e.class
+        close
+      end
+
+      def on_close
+        @log.trace { "closed fluent socket object_id=#{self.object_id}" }
+      end
+    end
+
+    class BaseInput < Fluent::Input
+      def initialize
+        super
+        require 'fluent/parser'
+      end
+
+      config_param :tag, :string
+      config_param :format, :string
+      config_param :port, :integer, :default => 5150
+      config_param :bind, :string, :default => '0.0.0.0'
+      config_param :source_host_key, :string, :default => nil
+
+      def configure(conf)
+        super
+
+        @parser = TextParser.new
+        @parser.configure(conf)
+      end
+
+      def start
+        @loop = Coolio::Loop.new
+        @handler = listen(method(:on_message))
+        @loop.attach(@handler)
+        @thread = Thread.new(&method(:run))
+      end
+
+      def shutdown
+        @loop.watchers.each { |w| w.detach }
+        @loop.stop
+        @handler.close
+        @thread.join
+      end
+
+      def run
+        @loop.run
+      rescue => e
+        log.error "unexpected error", :error => e, :error_class => e.class
+        log.error_backtrace
+      end
+
+      private
+
+      def on_message(msg, addr)
+        @parser.parse(msg) { |time, record|
+          unless time && record
+            log.warn "pattern not match: #{msg.inspect}"
+            return
+          end
+
+          record[@source_host_key] = addr[3] if @source_host_key
+          Engine.emit(@tag, time, record)
+        }
+      rescue => e
+        log.error msg.dump, :error => e, :error_class => e.class, :host => addr[3]
+        log.error_backtrace
+      end
+    end
   end
 end

data/lib/fluent/supervisor.rb

@@ -18,6 +18,7 @@
 
 require 'fluent/env'
 require 'fluent/log'
+require 'fluent/config'
 require 'etc'
 
 module Fluent
@@ -95,19 +96,22 @@ module Fluent
     end
 
     def initialize(opt)
+      @daemonize = opt[:daemonize]
       @config_path = opt[:config_path]
+      @inline_config = opt[:inline_config]
+      @use_v1_config = opt[:use_v1_config]
       @log_path = opt[:log_path]
-      @log_level = opt[:log_level]
-      @daemonize = opt[:daemonize]
-      @chgroup = opt[:chgroup]
-      @chuser = opt[:chuser]
+      @dry_run = opt[:dry_run]
       @libs = opt[:libs]
       @plugin_dirs = opt[:plugin_dirs]
-      @inline_config = opt[:inline_config]
+      @chgroup = opt[:chgroup]
+      @chuser = opt[:chuser]
+
+      apply_system_config(opt)
+
+      @log_level = opt[:log_level]
       @suppress_interval = opt[:suppress_interval]
-      @dry_run = opt[:dry_run]
       @suppress_config_dump = opt[:suppress_config_dump]
-      @use_v1_config = opt[:use_v1_config]
 
       log_opts = {:suppress_repeated_stacktrace => opt[:suppress_repeated_stacktrace]}
       @log = LoggerInitializer.new(@log_path, @log_level, @chuser, @chgroup, log_opts)
@@ -317,8 +321,9 @@ module Fluent
       end
     end
 
-    def read_config
-      $log.info "reading config file", :path=>@config_path
+    # with_log is for disabling logging before Log#init is called
+    def read_config(with_log = true)
+      $log.info "reading config file", :path => @config_path if with_log
       @config_fname = File.basename(@config_path)
       @config_basedir = File.dirname(@config_path)
       @config_data = File.read(@config_path)
@@ -329,8 +334,45 @@ module Fluent
       end
     end
 
+    class SystemConfig
+      include Configurable
+
+      config_param :log_level, :default => nil do |level|
+        Log.str_to_level(level)
+      end
+      config_param :suppress_repeated_stacktrace, :bool, :default => nil
+      config_param :emit_error_log_interval, :time, :default => nil
+      config_param :suppress_config_dump, :bool, :default => nil
+
+      def initialize(conf)
+        super()
+        configure(conf)
+      end
+
+      def to_opt
+        opt = {}
+        opt[:log_level] = @log_level unless @log_level.nil?
+        opt[:suppress_interval] = @emit_error_log_interval unless @emit_error_log_interval.nil?
+        opt[:suppress_config_dump] = @suppress_config_dump unless @suppress_config_dump.nil?
+        opt[:suppress_repeated_stacktrace] = @suppress_repeated_stacktrace unless @suppress_repeated_stacktrace.nil?
+        opt
+      end
+    end
+
+    def apply_system_config(opt)
+      read_config(false)
+      systems = Fluent::Config.parse(@config_data, @config_fname, @config_basedir, @use_v1_config).elements.select { |e|
+        e.name == 'system'
+      }
+      return if systems.empty?
+      raise ConfigError, "<system> is duplicated. <system> should be only one" if systems.size > 1
+
+      opt.merge!(SystemConfig.new(systems.first).to_opt)
+    end
+
     def run_configure
-      Fluent::Engine.parse_config(@config_data, @config_fname, @config_basedir, @use_v1_config)
+      conf = Fluent::Config.parse(@config_data, @config_fname, @config_basedir, @use_v1_config)
+      Fluent::Engine.run_configure(conf)
     end
 
     def change_privilege