fluentd 0.14.7-x64-mingw32 → 0.14.10-x64-mingw32

data/lib/fluent/plugin/in_http.rb

@@ -14,28 +14,29 @@
 #    limitations under the License.
 #
 
+require 'fluent/plugin/input'
+require 'fluent/plugin/parser'
+require 'fluent/event'
+
+require 'http/parser'
+require 'webrick/httputils'
 require 'uri'
 require 'socket'
 require 'json'
 
-require 'cool.io'
-
-require 'fluent/input'
-require 'fluent/event'
-require 'fluent/process'
+module Fluent::Plugin
+  class InHttpParser < Parser
+    Fluent::Plugin.register_parser('in_http', self)
+    def parse(text)
+      # this plugin is dummy implementation not to raise error
+      yield nil, nil
+    end
+  end
 
-module Fluent
   class HttpInput < Input
-    Plugin.register_input('http', self)
-
-    include DetachMultiProcessMixin
-
-    require 'http/parser'
+    Fluent::Plugin.register_input('http', self)
 
-    def initialize
-      require 'webrick/httputils'
-      super
-    end
+    helpers :parser, :compat_parameters, :event_loop
 
     EMPTY_GIF_IMAGE = "GIF89a\u0001\u0000\u0001\u0000\x80\xFF\u0000\xFF\xFF\xFF\u0000\u0000\u0000,\u0000\u0000\u0000\u0000\u0001\u0000\u0001\u0000\u0000\u0002\u0002D\u0001\u0000;".force_encoding("UTF-8")
 
@@ -52,25 +53,36 @@ module Fluent
     config_param :add_http_headers, :bool, default: false
     desc 'Add REMOTE_ADDR header to the record.'
     config_param :add_remote_addr, :bool, default: false
-    desc 'The format of the HTTP body.'
-    config_param :format, :string, default: 'default'
     config_param :blocking_timeout, :time, default: 0.5
     desc 'Set a white list of domains that can do CORS (Cross-Origin Resource Sharing)'
     config_param :cors_allow_origins, :array, default: nil
     desc 'Respond with empty gif image of 1x1 pixel.'
     config_param :respond_with_empty_img, :bool, default: false
 
+    config_section :parse do
+      config_set_default :@type, 'in_http'
+    end
+
+    EVENT_RECORD_PARAMETER = '_event_record'
+
     def configure(conf)
+      compat_parameters_convert(conf, :parser)
+
       super
 
-      m = if @format == 'default'
+      m = if @parser_configs.first['@type'] == 'in_http'
+            @parser_msgpack = parser_create(usage: 'parser_in_http_msgpack', type: 'msgpack')
+            @parser_msgpack.estimate_current_event = false
+            @parser_json = parser_create(usage: 'parser_in_http_json', type: 'json')
+            @parser_json.estimate_current_event = false
+            @format_name = 'default'
             method(:parse_params_default)
           else
-            @parser = Plugin.new_parser(@format)
-            @parser.configure(conf)
+            @parser = parser_create
+            @format_name = @parser_configs.first['@type']
             method(:parse_params_with_parser)
           end
-      (class << self; self; end).module_eval do
+      self.singleton_class.module_eval do
         define_method(:parse_params, m)
       end
     end
@@ -100,7 +112,11 @@ module Fluent
     end
 
     def start
-      log.debug "listening http on #{@bind}:#{@port}"
+      @_event_loop_run_timeout = @blocking_timeout
+
+      super
+
+      log.debug "listening http", bind: @bind, port: @port
 
       socket_manager_path = ENV['SERVERENGINE_SOCKETMANAGER_PATH']
       if Fluent.windows?
@@ -109,38 +125,24 @@ module Fluent
       client = ServerEngine::SocketManager::Client.new(socket_manager_path)
       lsock = client.listen_tcp(@bind, @port)
 
-      detach_multi_process do
-        super
-        @km = KeepaliveManager.new(@keepalive_timeout)
-        @lsock = Coolio::TCPServer.new(lsock, nil, Handler, @km, method(:on_request),
-                                       @body_size_limit, @format, log,
-                                       @cors_allow_origins)
-        @lsock.listen(@backlog) unless @backlog.nil?
-
-        @loop = Coolio::Loop.new
-        @loop.attach(@km)
-        @loop.attach(@lsock)
-
-        @thread = Thread.new(&method(:run))
-      end
+      @km = KeepaliveManager.new(@keepalive_timeout)
+      @lsock = Coolio::TCPServer.new(
+        lsock, nil, Handler, @km, method(:on_request),
+        @body_size_limit, @format_name, log,
+        @cors_allow_origins
+      )
+      @lsock.listen(@backlog) unless @backlog.nil?
+      event_loop_attach(@km)
+      event_loop_attach(@lsock)
+
+      @float_time_parser = Fluent::NumericTimeParser.new(:float)
     end
 
-    def shutdown
-      @loop.watchers.each {|w| w.detach }
-      @loop.stop
+    def close
       @lsock.close
-      @thread.join
-
       super
     end
 
-    def run
-      @loop.run(@blocking_timeout)
-    rescue
-      log.error "unexpected error", error: $!.to_s
-      log.error_backtrace
-    end
-
     def on_request(path_info, params)
       begin
         path = path_info[1..-1]  # remove /
@@ -170,9 +172,9 @@ module Fluent
         end
         time = if param_time = params['time']
                  param_time = param_time.to_f
-                 param_time.zero? ? Engine.now : Fluent::EventTime.from_time(Time.at(param_time))
+                 param_time.zero? ? Fluent::Engine.now : @float_time_parser.parse(param_time)
                else
-                 record_time.nil? ? Engine.now : record_time
+                 record_time.nil? ? Fluent::Engine.now : record_time
                end
       rescue
         return ["400 Bad Request", {'Content-Type'=>'text/plain'}, "400 Bad Request\n#{$!}\n"]
@@ -182,7 +184,7 @@ module Fluent
       begin
         # Support batched requests
         if record.is_a?(Array)
-          mes = MultiEventStream.new
+          mes = Fluent::MultiEventStream.new
           record.each do |single_record|
             if @add_http_headers
               params.each_pair { |k,v|
@@ -215,22 +217,23 @@ module Fluent
     private
 
     def parse_params_default(params)
-      record = if msgpack = params['msgpack']
-                 Engine.msgpack_factory.unpacker.feed(msgpack).read
-               elsif js = params['json']
-                 JSON.parse(js)
-               else
-                 raise "'json' or 'msgpack' parameter is required"
-               end
-      return nil, record
+      if msgpack = params['msgpack']
+        @parser_msgpack.parse(msgpack) do |_time, record|
+          return nil, record
+        end
+      elsif js = params['json']
+        @parser_json.parse(js) do |_time, record|
+          return nil, record
+        end
+      else
+        raise "'json' or 'msgpack' parameter is required"
+      end
     end
 
-    EVENT_RECORD_PARAMETER = '_event_record'
-
     def parse_params_with_parser(params)
       if content = params[EVENT_RECORD_PARAMETER]
         @parser.parse(content) { |time, record|
-          raise "Received event is not #{@format}: #{content}" if record.nil?
+          raise "Received event is not #{@format_name}: #{content}" if record.nil?
           return time, record
         }
       else
@@ -241,13 +244,13 @@ module Fluent
     class Handler < Coolio::Socket
       attr_reader :content_type
 
-      def initialize(io, km, callback, body_size_limit, format, log, cors_allow_origins)
+      def initialize(io, km, callback, body_size_limit, format_name, log, cors_allow_origins)
         super(io)
         @km = km
         @callback = callback
         @body_size_limit = body_size_limit
         @next_close = false
-        @format = format
+        @format_name = format_name
         @log = log
         @cors_allow_origins = cors_allow_origins
         @idle = 0
@@ -355,7 +358,7 @@ module Fluent
         uri = URI.parse(@parser.request_url)
         params = WEBrick::HTTPUtils.parse_query(uri.query)
 
-        if @format != 'default'
+        if @format_name != 'default'
           params[EVENT_RECORD_PARAMETER] = @body
         elsif @content_type =~ /^application\/x-www-form-urlencoded/
           params.update WEBrick::HTTPUtils.parse_query(@body)

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -18,8 +18,7 @@ require 'json'
 require 'webrick'
 require 'cgi'
 
-require 'cool.io'
-
+require 'fluent/config/types'
 require 'fluent/plugin/input'
 require 'fluent/plugin/output'
 require 'fluent/plugin/multi_output'
@@ -35,6 +34,7 @@ module Fluent::Plugin
     config_param :port, :integer, default: 24220
     config_param :tag, :string, default: nil
     config_param :emit_interval, :time, default: 60
+    config_param :include_config, :bool, default: true
 
     class MonitorServlet < WEBrick::HTTPServlet::AbstractServlet
       def initialize(server, agent)
@@ -78,12 +78,16 @@ module Fluent::Plugin
 
         # if ?debug=1 is set, set :with_debug_info for get_monitor_info
         # and :pretty_json for render_json_error
-        opts = {}
+        opts = {with_config: @agent.include_config}
         if s = qs['debug'] and s[0]
           opts[:with_debug_info] = true
           opts[:pretty_json] = true
         end
 
+        if with_config = get_search_parameter(qs, 'with_config'.freeze)
+          opts[:with_config] = Fluent::Config.bool_value(with_config)
+        end
+
         if tag = get_search_parameter(qs, 'tag'.freeze)
           # ?tag= to search an output plugin by match pattern
           if obj = @agent.plugin_info_by_tag(tag, opts)
@@ -329,7 +333,7 @@ module Fluent::Plugin
       obj['plugin_id'] = pe.plugin_id
       obj['plugin_category'] = plugin_category(pe)
       obj['type'] = pe.config['@type']
-      obj['config'] = pe.config if !opts.has_key?(:with_config) || opts[:with_config]
+      obj['config'] = pe.config if opts[:with_config]
 
       # run MONITOR_INFO in plugins' instance context and store the info to obj
       MONITOR_INFO.each_pair {|key,code|

data/lib/fluent/plugin/in_syslog.rb

@@ -25,8 +25,9 @@ module Fluent::Plugin
   class SyslogInput < Input
     Fluent::Plugin.register_input('syslog', self)
 
-    helpers :parser, :event_loop
+    helpers :parser, :compat_parameters, :event_loop
 
+    DEFAULT_PARSER = 'syslog'
     SYSLOG_REGEXP = /^\<([0-9]+)\>(.*)/
 
     FACILITY_MAP = {
@@ -79,48 +80,40 @@ module Fluent::Plugin
     desc 'The prefix of the tag. The tag itself is generated by the tag prefix, facility level, and priority.'
     config_param :tag, :string
     desc 'The transport protocol used to receive logs.(udp, tcp)'
-    config_param :protocol_type, default: :udp do |val|
-      case val.downcase
-      when 'tcp'
-        :tcp
-      when 'udp'
-        :udp
-      else
-        raise Fluent::ConfigError, "syslog input protocol type should be 'tcp' or 'udp'"
-      end
-    end
+    config_param :protocol_type, :enum, list: [:tcp, :udp], default: :udp
     desc 'If true, add source host to event record.'
     config_param :include_source_host, :bool, default: false
     desc 'Specify key of source host when include_source_host is true.'
     config_param :source_host_key, :string, default: 'source_host'.freeze
+    desc 'The field name of the priority.'
+    config_param :priority_key, :string, default: nil
+    desc 'The field name of the facility.'
+    config_param :facility_key, :string, default: nil
     config_param :blocking_timeout, :time, default: 0.5
     config_param :message_length_limit, :size, default: 2048
 
+    config_section :parse do
+      config_set_default :@type, DEFAULT_PARSER
+      config_param :with_priority, :bool, default: true
+    end
+
     def configure(conf)
+      compat_parameters_convert(conf, :parser)
+
       super
 
       @use_default = false
 
-      if conf.has_key?('format')
-        @parser = parser_create(usage: 'syslog_input', type: conf['format'], conf: conf)
-      else
-        conf['with_priority'] = true
-        @parser = parser_create(usage: 'syslog_input', type: 'syslog', conf: conf)
-        @use_default = true
-      end
+      @parser = parser_create
+      @parser_parse_priority = @parser.respond_to?(:with_priority) && @parser.with_priority
+
       @_event_loop_run_timeout = @blocking_timeout
     end
 
     def start
       super
 
-      callback = if @use_default
-                   method(:receive_data)
-                 else
-                   method(:receive_data_parser)
-                 end
-
-      @handler = listen(callback)
+      @handler = listen(method(:message_handler))
       event_loop_attach(@handler)
     end
 
@@ -132,42 +125,38 @@ module Fluent::Plugin
 
     private
 
-    def receive_data_parser(data, addr)
-      m = SYSLOG_REGEXP.match(data)
-      unless m
-        log.warn "invalid syslog message: #{data.dump}"
-        return
-      end
-      pri = m[1].to_i
-      text = m[2]
-
-      @parser.parse(text) { |time, record|
-        unless time && record
-          log.warn "pattern not match: #{text.inspect}"
+    def message_handler(data, addr)
+      pri = nil
+      text = data
+      unless @parser_parse_priority
+        m = SYSLOG_REGEXP.match(data)
+        unless m
+          log.warn "invalid syslog message: #{data.dump}"
           return
         end
+        pri = m[1].to_i
+        text = m[2]
+      end
 
-        record[@source_host_key] = addr[2] if @include_source_host
-        emit(pri, time, record)
-      }
-    rescue => e
-      log.error data.dump, error: e.to_s
-      log.error_backtrace
-    end
-
-    def receive_data(data, addr)
-      @parser.parse(data) { |time, record|
+      @parser.parse(text) do |time, record|
         unless time && record
-          log.warn "invalid syslog message", data: data
+          log.warn "failed to parse message", data: data
           return
         end
 
-        pri = record.delete('pri')
+        pri ||= record.delete('pri')
+        facility = FACILITY_MAP[pri >> 3]
+        priority = PRIORITY_MAP[pri & 0b111]
+
+        record[@priority_key] = priority if @priority_key
+        record[@facility_key] = facility if @facility_key
         record[@source_host_key] = addr[2] if @include_source_host
-        emit(pri, time, record)
-      }
+
+        tag = "#{@tag}.#{facility}.#{priority}"
+        emit(tag, time, record)
+      end
     rescue => e
-      log.error data.dump, error: e.to_s
+      log.error "invalid input", data: data, error: e
       log.error_backtrace
     end
 
@@ -190,12 +179,7 @@ module Fluent::Plugin
       end
     end
 
-    def emit(pri, time, record)
-      facility = FACILITY_MAP[pri >> 3]
-      priority = PRIORITY_MAP[pri & 0b111]
-
-      tag = "#{@tag}.#{facility}.#{priority}"
-
+    def emit(tag, time, record)
       router.emit(tag, time, record)
     rescue => e
       log.error "syslog failed to emit", error: e, tag: tag, record: Yajl.dump(record)

data/lib/fluent/plugin/in_tail.rb

@@ -164,6 +164,12 @@ module Fluent::Plugin
       super
     end
 
+    def close
+      super
+      # close file handles after all threads stopped (in #close of thread plugin helper)
+      close_watcher_handles
+    end
+
     def expand_paths
       date = Time.now
       paths = []
@@ -234,18 +240,27 @@ module Fluent::Plugin
 
     def stop_watchers(paths, immediate = false, unwatched = false)
       paths.each { |path|
-        tw = @tails.delete(path)
+        tw = @tails[path]
         if tw
           tw.unwatched = unwatched
           if immediate
-            close_watcher(tw, false)
+            detach_watcher(tw, false)
           else
-            close_watcher_after_rotate_wait(tw)
+            detach_watcher_after_rotate_wait(tw)
           end
         end
       }
     end
 
+    def close_watcher_handles
+      @tails.keys.each do |path|
+        tw = @tails.delete(path)
+        if tw
+          tw.close
+        end
+      end
+    end
+
     # refresh_watchers calls @tails.keys so we don't use stop_watcher -> start_watcher sequence for safety.
     def update_watcher(path, pe)
       if @pf
@@ -256,24 +271,25 @@ module Fluent::Plugin
       end
       rotated_tw = @tails[path]
       @tails[path] = setup_watcher(path, pe)
-      close_watcher_after_rotate_wait(rotated_tw) if rotated_tw
+      detach_watcher_after_rotate_wait(rotated_tw) if rotated_tw
     end
 
     # TailWatcher#close is called by another thread at shutdown phase.
     # It causes 'can't modify string; temporarily locked' error in IOHandler
     # so adding close_io argument to avoid this problem.
     # At shutdown, IOHandler's io will be released automatically after detached the event loop
-    def close_watcher(tw, close_io = true)
-      tw.close(close_io)
+    def detach_watcher(tw, close_io = true)
+      tw.detach
+      tw.close if close_io
       flush_buffer(tw)
       if tw.unwatched && @pf
         @pf[tw.path].update_pos(PositionFile::UNWATCHED_POSITION)
       end
     end
 
-    def close_watcher_after_rotate_wait(tw)
+    def detach_watcher_after_rotate_wait(tw)
       timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do
-        close_watcher(tw)
+        detach_watcher(tw)
       end
     end
 
@@ -435,14 +451,13 @@ module Fluent::Plugin
       def detach
         @timer_trigger.detach if @enable_watch_timer && @timer_trigger.attached?
         @stat_trigger.detach if @stat_trigger.attached?
+        @io_handler.on_notify if @io_handler
       end
 
-      def close(close_io = true)
-        if close_io && @io_handler
-          @io_handler.on_notify
+      def close
+        if @io_handler
           @io_handler.close
         end
-        detach
       end
 
       def on_notify
@@ -570,8 +585,8 @@ module Fluent::Plugin
                   else
                     @buffer << @io.readpartial(2048, @iobuf)
                   end
-                  while line = @buffer.slice!(/.*?\n/m)
-                    @lines << line
+                  while idx = @buffer.index("\n".freeze)
+                    @lines << @buffer.slice!(0, idx + 1)
                   end
                   if @lines.size >= @read_lines_limit
                     # not to use too much memory in case the file is very large