fluentd 0.12.40 → 1.6.2

data/lib/fluent/plugin/in_gc_stat.rb

@@ -14,13 +14,13 @@
 #    limitations under the License.
 #
 
-require 'cool.io'
+require 'fluent/plugin/input'
 
-require 'fluent/input'
+module Fluent::Plugin
+  class GCStatInput < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('gc_stat', self)
 
-module Fluent
-  class GCStatInput < Input
-    Plugin.register_input('gc_stat', self)
+    helpers :timer
 
     def initialize
       super
@@ -29,48 +29,26 @@ module Fluent
     config_param :emit_interval, :time, default: 60
     config_param :tag, :string
 
-    class TimerWatcher < Coolio::TimerWatcher
-      def initialize(interval, repeat, log, &callback)
-        @callback = callback
-        @log = log
-        super(interval, repeat)
-      end
-
-      def on_timer
-        @callback.call
-      rescue
-        # TODO log?
-        @log.error $!.to_s
-        @log.error_backtrace
-      end
-    end
-
     def configure(conf)
       super
     end
 
-    def start
-      @loop = Coolio::Loop.new
-      @timer = TimerWatcher.new(@emit_interval, true, log, &method(:on_timer))
-      @loop.attach(@timer)
-      @thread = Thread.new(&method(:run))
+    def multi_workers_ready?
+      true
     end
 
-    def shutdown
-      @loop.watchers.each {|w| w.detach }
-      @loop.stop
-      @thread.join
+    def start
+      super
+
+      timer_execute(:in_gc_stat, @emit_interval, &method(:on_timer))
     end
 
-    def run
-      @loop.run
-    rescue
-      log.error "unexpected error", error: $!.to_s
-      log.error_backtrace
+    def shutdown
+      super
     end
 
     def on_timer
-      now = Engine.now
+      now = Fluent::EventTime.now
       record = GC.stat
       router.emit(@tag, now, record)
     end

data/lib/fluent/plugin/in_http.rb

@@ -14,28 +14,29 @@
 #    limitations under the License.
 #
 
+require 'fluent/plugin/input'
+require 'fluent/plugin/parser'
+require 'fluent/event'
+
+require 'http/parser'
+require 'webrick/httputils'
 require 'uri'
 require 'socket'
 require 'json'
 
-require 'cool.io'
-
-require 'fluent/input'
-require 'fluent/event'
-require 'fluent/process'
+module Fluent::Plugin
+  class InHttpParser < Parser
+    Fluent::Plugin.register_parser('in_http', self)
+    def parse(text)
+      # this plugin is dummy implementation not to raise error
+      yield nil, nil
+    end
+  end
 
-module Fluent
   class HttpInput < Input
-    Plugin.register_input('http', self)
-
-    include DetachMultiProcessMixin
-
-    require 'http/parser'
+    Fluent::Plugin.register_input('http', self)
 
-    def initialize
-      require 'webrick/httputils'
-      super
-    end
+    helpers :parser, :compat_parameters, :event_loop, :server
 
     EMPTY_GIF_IMAGE = "GIF89a\u0001\u0000\u0001\u0000\x80\xFF\u0000\xFF\xFF\xFF\u0000\u0000\u0000,\u0000\u0000\u0000\u0000\u0001\u0000\u0001\u0000\u0000\u0002\u0002D\u0001\u0000;".force_encoding("UTF-8")
 
@@ -52,25 +53,42 @@ module Fluent
     config_param :add_http_headers, :bool, default: false
     desc 'Add REMOTE_ADDR header to the record.'
     config_param :add_remote_addr, :bool, default: false
-    desc 'The format of the HTTP body.'
-    config_param :format, :string, default: 'default'
     config_param :blocking_timeout, :time, default: 0.5
     desc 'Set a white list of domains that can do CORS (Cross-Origin Resource Sharing)'
     config_param :cors_allow_origins, :array, default: nil
     desc 'Respond with empty gif image of 1x1 pixel.'
     config_param :respond_with_empty_img, :bool, default: false
 
+    config_section :parse do
+      config_set_default :@type, 'in_http'
+    end
+
+    EVENT_RECORD_PARAMETER = '_event_record'
+
     def configure(conf)
+      compat_parameters_convert(conf, :parser)
+
       super
 
-      m = if @format == 'default'
+      m = if @parser_configs.first['@type'] == 'in_http'
+            @parser_msgpack = parser_create(usage: 'parser_in_http_msgpack', type: 'msgpack')
+            @parser_msgpack.estimate_current_event = false
+            @parser_json = parser_create(usage: 'parser_in_http_json', type: 'json')
+            @parser_json.estimate_current_event = false
+            @format_name = 'default'
+            @parser_time_key = if parser_config = conf.elements('parse').first
+                                 parser_config['time_key'] || 'time'
+                               else
+                                 'time'
+                               end
             method(:parse_params_default)
           else
-            @parser = Plugin.new_parser(@format)
-            @parser.configure(conf)
+            @parser = parser_create
+            @format_name = @parser_configs.first['@type']
+            @parser_time_key = @parser.time_key
             method(:parse_params_with_parser)
           end
-      (class << self; self; end).module_eval do
+      self.singleton_class.module_eval do
         define_method(:parse_params, m)
       end
     end
@@ -99,39 +117,27 @@ module Fluent
       end
     end
 
-    def start
-      log.debug "listening http on #{@bind}:#{@port}"
-      lsock = TCPServer.new(@bind, @port)
-
-      detach_multi_process do
-        super
-        @km = KeepaliveManager.new(@keepalive_timeout)
-        #@lsock = Coolio::TCPServer.new(@bind, @port, Handler, @km, method(:on_request), @body_size_limit)
-        @lsock = Coolio::TCPServer.new(lsock, nil, Handler, @km, method(:on_request),
-                                       @body_size_limit, @format, log,
-                                       @cors_allow_origins)
-        @lsock.listen(@backlog) unless @backlog.nil?
-
-        @loop = Coolio::Loop.new
-        @loop.attach(@km)
-        @loop.attach(@lsock)
-
-        @thread = Thread.new(&method(:run))
-      end
+    def multi_workers_ready?
+      true
     end
 
-    def shutdown
-      @loop.watchers.each {|w| w.detach }
-      @loop.stop
-      @lsock.close
-      @thread.join
+    def start
+      @_event_loop_run_timeout = @blocking_timeout
+
+      super
+
+      log.debug "listening http", bind: @bind, port: @port
+
+      @km = KeepaliveManager.new(@keepalive_timeout)
+      event_loop_attach(@km)
+
+      server_create_connection(:in_http, @port, bind: @bind, backlog: @backlog, &method(:on_server_connect))
+      @float_time_parser = Fluent::NumericTimeParser.new(:float)
     end
 
-    def run
-      @loop.run(@blocking_timeout)
-    rescue
-      log.error "unexpected error", error: $!.to_s
-      log.error_backtrace
+    def close
+      server_wait_until_stop
+      super
     end
 
     def on_request(path_info, params)
@@ -142,10 +148,11 @@ module Fluent
 
         # Skip nil record
         if record.nil?
+          log.debug { "incoming event is invalid: path=#{path_info} params=#{params.to_json}" }
           if @respond_with_empty_img
-            return ["200 OK", {'Content-type'=>'image/gif; charset=utf-8'}, EMPTY_GIF_IMAGE]
+            return ["200 OK", {'Content-Type'=>'image/gif; charset=utf-8'}, EMPTY_GIF_IMAGE]
           else
-            return ["200 OK", {'Content-type'=>'text/plain'}, ""]
+            return ["200 OK", {'Content-Type'=>'text/plain'}, ""]
           end
         end
 
@@ -162,20 +169,20 @@ module Fluent
           end
         end
         time = if param_time = params['time']
-                 param_time = param_time.to_i
-                 param_time.zero? ? Engine.now : param_time
+                 param_time = param_time.to_f
+                 param_time.zero? ? Fluent::Engine.now : @float_time_parser.parse(param_time)
                else
-                 record_time.nil? ? Engine.now : record_time
+                 record_time.nil? ? Fluent::Engine.now : record_time
                end
       rescue
-        return ["400 Bad Request", {'Content-type'=>'text/plain'}, "400 Bad Request\n#{$!}\n"]
+        return ["400 Bad Request", {'Content-Type'=>'text/plain'}, "400 Bad Request\n#{$!}\n"]
       end
 
       # TODO server error
       begin
         # Support batched requests
         if record.is_a?(Array)
-          mes = MultiEventStream.new
+          mes = Fluent::MultiEventStream.new
           record.each do |single_record|
             if @add_http_headers
               params.each_pair { |k,v|
@@ -187,7 +194,18 @@ module Fluent
             if @add_remote_addr
               single_record['REMOTE_ADDR'] = params['REMOTE_ADDR']
             end
-            single_time = single_record.delete("time") || time
+
+            if defined? @parser
+              single_time = @parser.parse_time(single_record)
+              single_time, single_record = @parser.convert_values(single_time, single_record)
+            else
+              single_time = if t = single_record.delete(@parser_time_key)
+                              Fluent::EventTime.from_time(Time.at(t))
+                            else
+                              time
+                            end
+            end
+
             mes.add(single_time, single_record)
           end
           router.emit_stream(tag, mes)
@@ -195,35 +213,52 @@ module Fluent
           router.emit(tag, time, record)
         end
       rescue
-        return ["500 Internal Server Error", {'Content-type'=>'text/plain'}, "500 Internal Server Error\n#{$!}\n"]
+        return ["500 Internal Server Error", {'Content-Type'=>'text/plain'}, "500 Internal Server Error\n#{$!}\n"]
       end
 
       if @respond_with_empty_img
-        return ["200 OK", {'Content-type'=>'image/gif; charset=utf-8'}, EMPTY_GIF_IMAGE]
+        return ["200 OK", {'Content-Type'=>'image/gif; charset=utf-8'}, EMPTY_GIF_IMAGE]
       else
-        return ["200 OK", {'Content-type'=>'text/plain'}, ""]
+        return ["200 OK", {'Content-Type'=>'text/plain'}, ""]
       end
     end
 
     private
 
-    def parse_params_default(params)
-      record = if msgpack = params['msgpack']
-                 Engine.msgpack_factory.unpacker.feed(msgpack).read
-               elsif js = params['json']
-                 JSON.parse(js)
-               else
-                 raise "'json' or 'msgpack' parameter is required"
-               end
-      return nil, record
+    def on_server_connect(conn)
+      handler = Handler.new(conn, @km, method(:on_request), @body_size_limit, @format_name, log, @cors_allow_origins)
+
+      conn.on(:data) do |data|
+        handler.on_read(data)
+      end
+
+      conn.on(:write_complete) do |_|
+        handler.on_write_complete
+      end
+
+      conn.on(:close) do |_|
+        handler.on_close
+      end
     end
 
-    EVENT_RECORD_PARAMETER = '_event_record'
+    def parse_params_default(params)
+      if msgpack = params['msgpack']
+        @parser_msgpack.parse(msgpack) do |_time, record|
+          return nil, record
+        end
+      elsif js = params['json']
+        @parser_json.parse(js) do |_time, record|
+          return nil, record
+        end
+      else
+        raise "'json' or 'msgpack' parameter is required"
+      end
+    end
 
     def parse_params_with_parser(params)
       if content = params[EVENT_RECORD_PARAMETER]
         @parser.parse(content) { |time, record|
-          raise "Received event is not #{@format}: #{content}" if record.nil?
+          raise "Received event is not #{@format_name}: #{content}" if record.nil?
           return time, record
         }
       else
@@ -231,20 +266,23 @@ module Fluent
       end
     end
 
-    class Handler < Coolio::Socket
-      def initialize(io, km, callback, body_size_limit, format, log, cors_allow_origins)
-        super(io)
+    class Handler
+      attr_reader :content_type
+
+      def initialize(io, km, callback, body_size_limit, format_name, log, cors_allow_origins)
+        @io = io
         @km = km
         @callback = callback
         @body_size_limit = body_size_limit
         @next_close = false
-        @format = format
+        @format_name = format_name
         @log = log
         @cors_allow_origins = cors_allow_origins
         @idle = 0
         @km.add(self)
 
-        @remote_port, @remote_addr = *Socket.unpack_sockaddr_in(io.getpeername) rescue nil
+        @remote_port, @remote_addr = io.remote_port, io.remote_addr
+        @parser = Http::Parser.new(self)
       end
 
       def step_idle
@@ -255,17 +293,13 @@ module Fluent
         @km.delete(self)
       end
 
-      def on_connect
-        @parser = Http::Parser.new(self)
-      end
-
       def on_read(data)
         @idle = 0
         @parser << data
       rescue
         @log.warn "unexpected error", error: $!.to_s
         @log.warn_backtrace
-        close
+        @io.close
       end
 
       def on_message_begin
@@ -283,6 +317,7 @@ module Fluent
         end
         @env = {}
         @content_type = ""
+        @content_encoding = ""
         headers.each_pair {|k,v|
           @env["HTTP_#{k.gsub('-','_').upcase}"] = v
           case k
@@ -292,6 +327,8 @@ module Fluent
             size = v.to_i
           when /Content-Type/i
             @content_type = v
+          when /Content-Encoding/i
+            @content_encoding = v
           when /Connection/i
             if v =~ /close/i
               @keep_alive = false
@@ -304,6 +341,10 @@ module Fluent
             # For multiple X-Forwarded-For headers. Use first header value.
             v = v.first if v.is_a?(Array)
             @remote_addr = v.split(",").first
+          when /Access-Control-Request-Method/i
+            @access_control_request_method = v
+          when /Access-Control-Request-Headers/i
+            @access_control_request_headers = v
           end
         }
         if expect
@@ -329,26 +370,76 @@ module Fluent
         @body << chunk
       end
 
+      # Web browsers can send an OPTIONS request before performing POST
+      # to check if cross-origin requests are supported.
+      def handle_options_request
+        # Is CORS enabled in the first place?
+        if @cors_allow_origins.nil?
+          return send_response_and_close("403 Forbidden", {}, "")
+        end
+
+        # in_http does not support HTTP methods except POST
+        if @access_control_request_method != 'POST'
+          return send_response_and_close("403 Forbidden", {}, "")
+        end
+
+        header = {
+          "Access-Control-Allow-Methods" => "POST",
+          "Access-Control-Allow-Headers" => @access_control_request_headers || "",
+        }
+
+        # Check the origin and send back a CORS response
+        if @cors_allow_origins.include?('*')
+          header["Access-Control-Allow-Origin"] = "*"
+          send_response_and_close("200 OK", header, "")
+        elsif include_cors_allow_origin
+          header["Access-Control-Allow-Origin"] = @origin
+          send_response_and_close("200 OK", header, "")
+        else
+          send_response_and_close("403 Forbidden", {}, "")
+        end
+      end
+
       def on_message_complete
         return if closing?
 
+        if @parser.http_method == 'OPTIONS'
+          return handle_options_request()
+        end
+
         # CORS check
         # ==========
         # For every incoming request, we check if we have some CORS
         # restrictions and white listed origins through @cors_allow_origins.
         unless @cors_allow_origins.nil?
-          unless @cors_allow_origins.include?(@origin)
+          unless @cors_allow_origins.include?('*') or include_cors_allow_origin
             send_response_and_close("403 Forbidden", {'Connection' => 'close'}, "")
             return
           end
         end
 
+        # Content Encoding
+        # =================
+        # Decode payload according to the "Content-Encoding" header.
+        # For now, we only support 'gzip' and 'deflate'.
+        begin
+          if @content_encoding == 'gzip'
+            @body = Zlib::GzipReader.new(StringIO.new(@body)).read
+          elsif @content_encoding == 'deflate'
+            @body = Zlib::Inflate.inflate(@body)
+          end
+        rescue
+          @log.warn 'fails to decode payload', error: $!.to_s
+          send_response_and_close("400 Bad Request", {}, "")
+          return
+        end
+
         @env['REMOTE_ADDR'] = @remote_addr if @remote_addr
 
         uri = URI.parse(@parser.request_url)
         params = WEBrick::HTTPUtils.parse_query(uri.query)
 
-        if @format != 'default'
+        if @format_name != 'default'
           params[EVENT_RECORD_PARAMETER] = @body
         elsif @content_type =~ /^application\/x-www-form-urlencoded/
           params.update WEBrick::HTTPUtils.parse_query(@body)
@@ -368,7 +459,14 @@ module Fluent
         code, header, body = *@callback.call(path_info, params)
         body = body.to_s
 
-        header['Access-Control-Allow-Origin'] = @origin if !@cors_allow_origins.nil? && @cors_allow_origins.include?(@origin)
+        unless @cors_allow_origins.nil?
+          if @cors_allow_origins.include?('*')
+            header['Access-Control-Allow-Origin'] = '*'
+          elsif include_cors_allow_origin
+            header['Access-Control-Allow-Origin'] = @origin
+          end
+        end
+
         if @keep_alive
           header['Connection'] = 'Keep-Alive'
           send_response(code, header, body)
@@ -377,8 +475,12 @@ module Fluent
         end
       end
 
+      def close
+        @io.close
+      end
+
       def on_write_complete
-        close if @next_close
+        @io.close if @next_close
       end
 
       def send_response_and_close(code, header, body)
@@ -391,17 +493,17 @@ module Fluent
       end
 
       def send_response(code, header, body)
-        header['Content-length'] ||= body.bytesize
-        header['Content-type'] ||= 'text/plain'
+        header['Content-Length'] ||= body.bytesize
+        header['Content-Type'] ||= 'text/plain'
 
         data = %[HTTP/1.1 #{code}\r\n]
         header.each_pair {|k,v|
           data << "#{k}: #{v}\r\n"
         }
         data << "\r\n"
-        write data
+        @io.write(data)
 
-        write body
+        @io.write(body)
       end
 
       def send_response_nobody(code, header)
@@ -410,7 +512,18 @@ module Fluent
           data << "#{k}: #{v}\r\n"
         }
         data << "\r\n"
-        write data
+        @io.write(data)
+      end
+
+      def include_cors_allow_origin
+        if @cors_allow_origins.include?(@origin)
+          return true
+        end
+        filtered_cors_allow_origins = @cors_allow_origins.select {|origin| origin != ""}
+        return filtered_cors_allow_origins.find do |origin|
+          (start_str,end_str) = origin.split("*",2)
+          @origin.start_with?(start_str) and @origin.end_with?(end_str)
+        end != nil
       end
     end
   end