fluentd 0.12.43 → 0.14.0

data/lib/fluent/plugin/in_gc_stat.rb

@@ -50,6 +50,8 @@ module Fluent
     end
 
     def start
+      super
+
       @loop = Coolio::Loop.new
       @timer = TimerWatcher.new(@emit_interval, true, log, &method(:on_timer))
       @loop.attach(@timer)
@@ -60,6 +62,8 @@ module Fluent
       @loop.watchers.each {|w| w.detach }
       @loop.stop
       @thread.join
+
+      super
     end
 
     def run

data/lib/fluent/plugin/in_http.rb

@@ -101,12 +101,17 @@ module Fluent
 
     def start
       log.debug "listening http on #{@bind}:#{@port}"
-      lsock = TCPServer.new(@bind, @port)
+
+      socket_manager_path = ENV['SERVERENGINE_SOCKETMANAGER_PATH']
+      if Fluent.windows?
+        socket_manager_path = socket_manager_path.to_i
+      end
+      client = ServerEngine::SocketManager::Client.new(socket_manager_path)
+      lsock = client.listen_tcp(@bind, @port)
 
       detach_multi_process do
         super
         @km = KeepaliveManager.new(@keepalive_timeout)
-        #@lsock = Coolio::TCPServer.new(@bind, @port, Handler, @km, method(:on_request), @body_size_limit)
         @lsock = Coolio::TCPServer.new(lsock, nil, Handler, @km, method(:on_request),
                                        @body_size_limit, @format, log,
                                        @cors_allow_origins)
@@ -125,6 +130,8 @@ module Fluent
       @loop.stop
       @lsock.close
       @thread.join
+
+      super
     end
 
     def run
@@ -143,9 +150,9 @@ module Fluent
         # Skip nil record
         if record.nil?
           if @respond_with_empty_img
-            return ["200 OK", {'Content-type'=>'image/gif; charset=utf-8'}, EMPTY_GIF_IMAGE]
+            return ["200 OK", {'Content-Type'=>'image/gif; charset=utf-8'}, EMPTY_GIF_IMAGE]
           else
-            return ["200 OK", {'Content-type'=>'text/plain'}, ""]
+            return ["200 OK", {'Content-Type'=>'text/plain'}, ""]
           end
         end
 
@@ -162,13 +169,13 @@ module Fluent
           end
         end
         time = if param_time = params['time']
-                 param_time = param_time.to_i
-                 param_time.zero? ? Engine.now : param_time
+                 param_time = param_time.to_f
+                 param_time.zero? ? Engine.now : Fluent::EventTime.from_time(Time.at(param_time))
                else
                  record_time.nil? ? Engine.now : record_time
                end
       rescue
-        return ["400 Bad Request", {'Content-type'=>'text/plain'}, "400 Bad Request\n#{$!}\n"]
+        return ["400 Bad Request", {'Content-Type'=>'text/plain'}, "400 Bad Request\n#{$!}\n"]
       end
 
       # TODO server error
@@ -195,13 +202,13 @@ module Fluent
           router.emit(tag, time, record)
         end
       rescue
-        return ["500 Internal Server Error", {'Content-type'=>'text/plain'}, "500 Internal Server Error\n#{$!}\n"]
+        return ["500 Internal Server Error", {'Content-Type'=>'text/plain'}, "500 Internal Server Error\n#{$!}\n"]
       end
 
       if @respond_with_empty_img
-        return ["200 OK", {'Content-type'=>'image/gif; charset=utf-8'}, EMPTY_GIF_IMAGE]
+        return ["200 OK", {'Content-Type'=>'image/gif; charset=utf-8'}, EMPTY_GIF_IMAGE]
       else
-        return ["200 OK", {'Content-type'=>'text/plain'}, ""]
+        return ["200 OK", {'Content-Type'=>'text/plain'}, ""]
       end
     end
 
@@ -232,6 +239,8 @@ module Fluent
     end
 
     class Handler < Coolio::Socket
+      attr_reader :content_type
+
       def initialize(io, km, callback, body_size_limit, format, log, cors_allow_origins)
         super(io)
         @km = km
@@ -300,10 +309,6 @@ module Fluent
             end
           when /Origin/i
             @origin  = v
-          when /X-Forwarded-For/i
-            # For multiple X-Forwarded-For headers. Use first header value.
-            v = v.first if v.is_a?(Array)
-            @remote_addr = v.split(",").first
           end
         }
         if expect
@@ -357,8 +362,6 @@ module Fluent
           params.update WEBrick::HTTPUtils.parse_form_data(@body, boundary)
         elsif @content_type =~ /^application\/json/
           params['json'] = @body
-        elsif @content_type =~ /^application\/msgpack/
-          params['msgpack'] = @body
         end
         path_info = uri.path
 
@@ -391,8 +394,8 @@ module Fluent
       end
 
       def send_response(code, header, body)
-        header['Content-length'] ||= body.bytesize
-        header['Content-type'] ||= 'text/plain'
+        header['Content-Length'] ||= body.bytesize
+        header['Content-Type'] ||= 'text/plain'
 
         data = %[HTTP/1.1 #{code}\r\n]
         header.each_pair {|k,v|

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -32,9 +32,6 @@ module Fluent
     config_param :port, :integer, default: 24220
     config_param :tag, :string, default: nil
     config_param :emit_interval, :time, default: 60
-    config_param :emit_config, :bool, default: false
-    config_param :include_config, :bool, default: true
-    config_param :include_retry, :bool, default: true
 
     class MonitorServlet < WEBrick::HTTPServlet::AbstractServlet
       def initialize(server, agent)
@@ -78,24 +75,12 @@ module Fluent
 
         # if ?debug=1 is set, set :with_debug_info for get_monitor_info
         # and :pretty_json for render_json_error
-        opts = {with_config: @agent.include_config, with_retry: @agent.include_retry}
+        opts = {}
         if s = qs['debug'] and s[0]
           opts[:with_debug_info] = true
           opts[:pretty_json] = true
         end
 
-        if ivars = (qs['with_ivars'] || []).first
-          opts[:ivars] = ivars.split(',')
-        end
-
-        if with_config = get_search_parameter(qs, 'with_config'.freeze)
-          opts[:with_config] = Fluent::Config.bool_value(with_config)
-        end
-
-        if with_retry = get_search_parameter(qs, 'with_retry'.freeze)
-          opts[:with_retry] = Fluent::Config.bool_value(with_retry)
-        end
-
         if tag = get_search_parameter(qs, 'tag'.freeze)
           # ?tag= to search an output plugin by match pattern
           if obj = @agent.plugin_info_by_tag(tag, opts)
@@ -250,6 +235,8 @@ module Fluent
     end
 
     def start
+      super
+
       log.debug "listening monitoring http server on http://#{@bind}:#{@port}/api/plugins"
       @srv = WEBrick::HTTPServer.new({
           BindAddress: @bind,
@@ -268,7 +255,7 @@ module Fluent
         log.debug "tag parameter is specified. Emit plugins info to '#{@tag}'"
 
         @loop = Coolio::Loop.new
-        opts = {with_config: @emit_config, with_retry: false}
+        opts = {with_config: false}
         timer = TimerWatcher.new(@emit_interval, log) {
           es = MultiEventStream.new
           now = Engine.now
@@ -305,12 +292,14 @@ module Fluent
         @thread_for_emit.join
         @thread_for_emit = nil
       end
+
+      super
     end
 
     MONITOR_INFO = {
-      'output_plugin' => 'is_a?(::Fluent::Output)', # deprecated. Use plugin_category instead
-      'buffer_queue_length' => '@buffer.queue_size',
-      'buffer_total_queued_size' => '@buffer.total_queued_chunk_size',
+      'output_plugin' => 'is_a?(::Fluent::Plugin::Output)',
+      'buffer_queue_length' => '@buffer.queue.size',
+      'buffer_total_queued_size' => '@buffer.stage_size + @buffer.queue_size',
       'retry_count' => '@num_errors',
     }
 
@@ -379,7 +368,7 @@ module Fluent
     # multiple plugins could have the same type
     def plugins_info_by_type(type, opts={})
       array = all_plugins.select {|pe|
-        (pe.config['@type'] == type || pe.config['type'] == type || Fluent::Plugin.lookup_name_from_class(pe.class)) rescue nil
+        (pe.config['@type'] == type) rescue nil
       }
       array.map {|pe|
         get_monitor_info(pe, opts)
@@ -394,7 +383,6 @@ module Fluent
 
     # TODO: use %i() after drop ruby v1.9.3 support.
     IGNORE_ATTRIBUTES = %W(@config_root_section @config @masked_config).map(&:to_sym)
-    EMPTY_RESULT = {}
 
     # get monitor info from the plugin `pe` and return a hash object
     def get_monitor_info(pe, opts={})
@@ -403,8 +391,8 @@ module Fluent
       # Common plugin information
       obj['plugin_id'] = pe.plugin_id
       obj['plugin_category'] = plugin_category(pe)
-      obj['type'] = pe.config['@type'] || pe.config['type'] || Fluent::Plugin.lookup_name_from_class(pe.class)
-      obj['config'] = pe.config if opts[:with_config]
+      obj['type'] = pe.config['@type']
+      obj['config'] = pe.config if !opts.has_key?(:with_config) || opts[:with_config]
 
       # run MONITOR_INFO in plugins' instance context and store the info to obj
       MONITOR_INFO.each_pair {|key,code|
@@ -414,13 +402,6 @@ module Fluent
         end
       }
 
-      if opts[:with_retry]
-        num_errors = pe.instance_variable_get(:@num_errors)
-        if num_errors
-          obj['retry'] = num_errors.zero? ? EMPTY_RESULT : get_retry_info(pe, num_errors)
-        end
-      end
-
       # include all instance variables if :with_debug_info is set
       if opts[:with_debug_info]
         iv = {}
@@ -432,32 +413,18 @@ module Fluent
           }
         end
         obj['instance_variables'] = iv
-      elsif ivars = opts[:ivars]
-        iv = {}
-        ivars.each {|name|
-          iname = "@#{name}"
-          iv[name] = pe.instance_variable_get(iname) if pe.instance_variable_defined?(iname)
-        }
-        obj['instance_variables'] = iv
       end
 
       obj
     end
 
-    def get_retry_info(pe, num_errors)
-      retry_variables = {}
-      retry_variables['steps'] = num_errors
-      retry_variables['next_time'] = Time.at(pe.instance_variable_get('@next_retry_time'.freeze))
-      retry_variables
-    end
-
     def plugin_category(pe)
       case pe
-      when Fluent::Input
+      when Fluent::Plugin::Input
         'input'.freeze
-      when Fluent::Output
+      when Fluent::Plugin::Output, Fluent::Plugin::BareOutput
         'output'.freeze
-      when Fluent::Filter
+      when Fluent::Plugin::Filter
         'filter'.freeze
       else
         'unknown'.freeze

data/lib/fluent/plugin/in_object_space.rb

@@ -52,6 +52,8 @@ module Fluent
     end
 
     def start
+      super
+
       @loop = Coolio::Loop.new
       @timer = TimerWatcher.new(@emit_interval, true, log, &method(:on_timer))
       @loop.attach(@timer)
@@ -62,6 +64,8 @@ module Fluent
       @loop.watchers.each {|w| w.detach }
       @loop.stop
       @thread.join
+
+      super
     end
 
     def run
@@ -115,7 +119,8 @@ module Fluent
 
       router.emit(@tag, now, record)
     rescue => e
-      log.error "object space failed to emit", error: e.to_s, error_class: e.class.to_s, tag: @tag, record: Yajl.dump(record)
+      log.error "object space failed to emit", error: e, tag: @tag, record: Yajl.dump(record)
+      log.error_backtrace
     end
   end
 end

data/lib/fluent/plugin/in_stream.rb

@@ -35,6 +35,8 @@ module Fluent
     end
 
     def start
+      super
+
       @loop = Coolio::Loop.new
       @lsock = listen
       @loop.attach(@lsock)
@@ -46,6 +48,8 @@ module Fluent
       @loop.stop
       @lsock.close
       @thread.join
+
+      super
     end
 
     #def listen
@@ -96,8 +100,8 @@ module Fluent
         entries.each {|e|
           record = e[1]
           next if record.nil?
-          time = e[0].to_i
-          time = (now ||= Engine.now) if time == 0
+          time = e[0]
+          time = (now ||= Engine.now) if time.to_i == 0
           es.add(time, record)
         }
         router.emit_stream(tag, es)
@@ -108,7 +112,7 @@ module Fluent
         return if record.nil?
 
         time = msg[1]
-        time = Engine.now if time == 0
+        time = Engine.now if time.to_i == 0
         router.emit(tag, time, record)
       end
     end

data/lib/fluent/plugin/in_syslog.rb

@@ -88,72 +88,36 @@ module Fluent
       end
     end
     desc 'If true, add source host to event record.'
-    config_param :include_source_host, :bool, default: false, deprecated: "use source_hostname_key instead"
+    config_param :include_source_host, :bool, default: false
     desc 'Specify key of source host when include_source_host is true.'
-    config_param :source_host_key, :string, default: 'source_host'.freeze, deprecated: "use source_hostname_key instead"
-    desc "The field name of the client's hostname."
-    config_param :source_hostname_key, :string, default: nil
-    desc "The field name of the client's source address."
-    config_param :source_address_key, :string, default: nil
-    desc 'Try to resolve hostname from IP addresses or not.'
-    config_param :resolve_hostname, :bool, default: nil
-    desc 'The field name of the priority.'
-    config_param :priority_key, :string, default: nil
-    desc 'The field name of the facility.'
-    config_param :facility_key, :string, default: nil
-    desc "The max bytes of message"
-    config_param :message_length_limit, :size, default: 2048
+    config_param :source_host_key, :string, default: 'source_host'.freeze
     config_param :blocking_timeout, :time, default: 0.5
 
-    desc 'If true, accept syslog message without PRI part'
-    config_param :allow_without_priority, :bool, default: false
-    # 13 is the default value of rsyslog and syslog-ng
-    desc 'The default PRI value (0 - 191 are available)'
-    config_param :default_priority, :integer, default: 13
-
     def configure(conf)
       super
 
-      if @default_priority < 0 && @default_priority > 191
-        raise ConfigError, "default_priority must be 0 ~ 191"
-      end
-
-      if @allow_without_priority && conf['message_format'] == 'auto'
-        raise ConfigError, "message_format auto isn't allowed when allow_without_priority is true"
-      end
+      @use_default = false
 
-      conf['with_priority'] = !@allow_without_priority
       if conf.has_key?('format')
         @parser = Plugin.new_parser(conf['format'])
         @parser.configure(conf)
       else
+        conf['with_priority'] = true
         @parser = TextParser::SyslogParser.new
         @parser.configure(conf)
         @use_default = true
       end
-
-      if @source_hostname_key.nil? && @include_source_host
-        @source_hostname_key = @source_host_key
-      end
-      if @source_hostname_key
-        if @resolve_hostname.nil?
-          @resolve_hostname = true
-        elsif !@resolve_hostname # user specifies "false" in configure
-          raise Fluent::ConfigError, "resolve_hostname must be true with source_hostname_key"
-        end
-      end
     end
 
     def start
-      callback = if @allow_without_priority
-                   method(:receive_data_allow_without_priority)
+      super
+
+      callback = if @use_default
+                   method(:receive_data)
                  else
-                   if @use_default
-                     method(:receive_data_default)
-                   else
-                     method(:receive_data_with_format)
-                   end
+                   method(:receive_data_parser)
                  end
+
       @loop = Coolio::Loop.new
       @handler = listen(callback)
       @loop.attach(@handler)
@@ -166,6 +130,8 @@ module Fluent
       @loop.stop
       @handler.close
       @thread.join
+
+      super
     end
 
     def run
@@ -177,16 +143,7 @@ module Fluent
 
     private
 
-    ## SyslogParser with PRI part
-    def receive_data_default(data, addr)
-      parse_text(data, addr)
-    rescue => e
-      log.error data.dump, error: e.to_s
-      log.error_backtrace
-    end
-
-    ## PRI part parser + custom parser without PRI part
-    def receive_data_with_format(data, addr)
+    def receive_data_parser(data, addr)
       m = SYSLOG_REGEXP.match(data)
       unless m
         log.warn "invalid syslog message: #{data.dump}"
@@ -195,70 +152,64 @@ module Fluent
       pri = m[1].to_i
       text = m[2]
 
-      parse_text(text, addr, pri)
-    rescue => e
-      log.error data.dump, error: e.to_s
-      log.error_backtrace
-    end
-
-    ## PRI part parser + SyslogParser without PRI part | custom parser without PRI part
-    def receive_data_allow_without_priority(data, addr)
-      m = SYSLOG_REGEXP.match(data)
-      if m
-        pri = m[1].to_i
-        text = m[2]
-      else
-        pri = @default_priority
-        text = data
-      end
+      @parser.parse(text) { |time, record|
+        unless time && record
+          log.warn "pattern not match: #{text.inspect}"
+          return
+        end
 
-      parse_text(text, addr, pri)
+        record[@source_host_key] = addr[2] if @include_source_host
+        emit(pri, time, record)
+      }
     rescue => e
       log.error data.dump, error: e.to_s
       log.error_backtrace
     end
 
-    def parse_text(text, addr, pri = nil)
-      @parser.parse(text) { |time, record|
+    def receive_data(data, addr)
+      @parser.parse(data) { |time, record|
         unless time && record
-          log.warn "pattern not match: #{text.inspect}"
+          log.warn "invalid syslog message", data: data
           return
         end
 
-        ## from receive_data_default
-        pri = record.delete('pri'.freeze) unless pri
-
-        facility = FACILITY_MAP[pri >> 3]
-        priority = PRIORITY_MAP[pri & 0b111]
-
-        record[@priority_key] = priority if @priority_key
-        record[@facility_key] = facility if @facility_key
-        record[@source_hostname_key] = addr[2] if @source_hostname_key
-        record[@source_address_key] = addr[3] if @source_address_key
-
-        tag = "#{@tag}.#{facility}.#{priority}"
-        emit(tag, time, record)
+        pri = record.delete('pri')
+        record[@source_host_key] = addr[2] if @include_source_host
+        emit(pri, time, record)
       }
+    rescue => e
+      log.error data.dump, error: e.to_s
+      log.error_backtrace
     end
 
     private
 
     def listen(callback)
       log.info "listening syslog socket on #{@bind}:#{@port} with #{@protocol_type}"
+      socket_manager_path = ENV['SERVERENGINE_SOCKETMANAGER_PATH']
+      if Fluent.windows?
+        socket_manager_path = socket_manager_path.to_i
+      end
+      client = ServerEngine::SocketManager::Client.new(socket_manager_path)
       if @protocol_type == :udp
-        @usock = SocketUtil.create_udp_socket(@bind)
-        @usock.bind(@bind, @port)
-        SocketUtil::UdpHandler.new(@usock, log, @message_length_limit, callback, @resolve_hostname)
+        @usock = client.listen_udp(@bind, @port)
+        SocketUtil::UdpHandler.new(@usock, log, 2048, callback)
       else
         # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
-        Coolio::TCPServer.new(@bind, @port, SocketUtil::TcpHandler, log, "\n", callback, @resolve_hostname)
+        lsock = client.listen_tcp(@bind, @port)
+        Coolio::TCPServer.new(lsock, nil, SocketUtil::TcpHandler, log, "\n", callback)
       end
     end
 
-    def emit(tag, time, record)
+    def emit(pri, time, record)
+      facility = FACILITY_MAP[pri >> 3]
+      priority = PRIORITY_MAP[pri & 0b111]
+
+      tag = "#{@tag}.#{facility}.#{priority}"
+
       router.emit(tag, time, record)
     rescue => e
-      log.error "syslog failed to emit", error: e.to_s, error_class: e.class.to_s, tag: tag, record: Yajl.dump(record)
+      log.error "syslog failed to emit", error: e, tag: tag, record: Yajl.dump(record)
     end
   end
 end