fluentd 0.14.8 → 0.14.9

data/lib/fluent/plugin/in_forward.rb

@@ -14,23 +14,26 @@
 #    limitations under the License.
 #
 
-require 'fcntl'
 
-require 'cool.io'
+require 'fluent/plugin/input'
+require 'fluent/msgpack_factory'
 require 'yajl'
+require 'digest'
 
-require 'fluent/input'
+require 'fluent/plugin/socket_util'
+require 'fcntl'
+require 'cool.io'
 
-module Fluent
+module Fluent::Plugin
   class ForwardInput < Input
-    Plugin.register_input('forward', self)
+    Fluent::Plugin.register_input('forward', self)
 
-    LISTEN_PORT = 24224
+    # See the wiki page below for protocol specification
+    # https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1
 
-    def initialize
-      super
-      require 'fluent/plugin/socket_util'
-    end
+    helpers :event_loop
+
+    LISTEN_PORT = 24224
 
     desc 'The port to listen to.'
     config_param :port, :integer, default: LISTEN_PORT
@@ -125,13 +128,12 @@ module Fluent
             })
         end
       end
+      @lsock = @usock = nil
     end
 
     def start
       super
 
-      @loop = Coolio::Loop.new
-
       socket_manager_path = ENV['SERVERENGINE_SOCKETMANAGER_PATH']
       if Fluent.windows?
         socket_manager_path = socket_manager_path.to_i
@@ -139,32 +141,17 @@ module Fluent
       client = ServerEngine::SocketManager::Client.new(socket_manager_path)
 
       @lsock = listen(client)
-      @loop.attach(@lsock)
+      event_loop_attach(@lsock)
 
       @usock = client.listen_udp(@bind, @port)
       @usock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK)
       @hbr = HeartbeatRequestHandler.new(@usock, method(:on_heartbeat_request))
-      @loop.attach(@hbr)
-
-      @thread = Thread.new(&method(:run))
+      event_loop_attach(@hbr)
     end
 
-    def shutdown
-      # In test cases it occasionally appeared that when detaching a watcher, another watcher is also detached.
-      # In the case in the iteration of watchers, a watcher that has been already detached is intended to be detached
-      # and therfore RuntimeError occurs saying that it is not attached to a loop.
-      # It occurs only when testing for sending responses to ForwardOutput.
-      # Sending responses needs to write the socket that is previously used only to read
-      # and a handler has 2 watchers that is used to read and to write.
-      # This problem occurs possibly because those watchers are thought to be related to each other
-      # and when detaching one of them the other is also detached for some reasons.
-      # As a workaround, check if watchers are attached before detaching them.
-      @loop.watchers.each {|w| w.detach if w.attached? }
-      @loop.stop
-      @usock.close
-      @thread.join
-      @lsock.close
-
+    def close
+      @lsock.close if @lsock
+      @usock.close if @usock
       super
     end
 
@@ -176,23 +163,6 @@ module Fluent
       s
     end
 
-    #config_param :path, :string, :default => DEFAULT_SOCKET_PATH
-    #def listen
-    #  if File.exist?(@path)
-    #    File.unlink(@path)
-    #  end
-    #  FileUtils.mkdir_p File.dirname(@path)
-    #  log.debug "listening fluent socket on #{@path}"
-    #  Coolio::UNIXServer.new(@path, Handler, method(:on_message))
-    #end
-
-    def run
-      @loop.run(@blocking_timeout)
-    rescue => e
-      log.error "unexpected error", error: e
-      log.error_backtrace
-    end
-
     private
 
     def handle_connection(conn)
@@ -288,29 +258,6 @@ module Fluent
       nil
     end
 
-    # message Entry {
-    #   1: long time
-    #   2: object record
-    # }
-    #
-    # message Forward {
-    #   1: string tag
-    #   2: list<Entry> entries
-    #   3: object option (optional)
-    # }
-    #
-    # message PackedForward {
-    #   1: string tag
-    #   2: raw entries  # msgpack stream of Entry
-    #   3: object option (optional)
-    # }
-    #
-    # message Message {
-    #   1: string tag
-    #   2: long? time
-    #   3: object record
-    #   4: object option (optional)
-    # }
     def on_message(msg, chunk_size, peeraddr)
       if msg.nil?
         # for future TCP heartbeat_request
@@ -338,7 +285,7 @@ module Fluent
         # PackedForward
         option = msg[2]
         size = (option && option['size']) || 0
-        es_class = (option && option['compressed'] == 'gzip') ? CompressedMessagePackEventStream : MessagePackEventStream
+        es_class = (option && option['compressed'] == 'gzip') ? Fluent::CompressedMessagePackEventStream : Fluent::MessagePackEventStream
         es = es_class.new(entries, nil, size.to_i)
         es = check_and_skip_invalid_event(tag, es, peeraddr) if @skip_invalid_event
         es = add_source_host(es, peeraddr[2]) if @source_hostname_key
@@ -349,12 +296,12 @@ module Fluent
         es = if @skip_invalid_event
                check_and_skip_invalid_event(tag, entries, peeraddr)
              else
-               es = MultiEventStream.new
+               es = Fluent::MultiEventStream.new
                entries.each { |e|
                  record = e[1]
                  next if record.nil?
                  time = e[0]
-                 time = (now ||= Engine.now) if time.to_i == 0
+                 time = Fluent::Engine.now if time.nil? || time.to_i == 0 # `to_i == 0` for empty EventTime
                  es.add(time, record)
                }
                es
@@ -372,7 +319,7 @@ module Fluent
           return msg[3] # retry never succeeded so return ack and drop incoming event.
         end
         return if record.nil?
-        time = Engine.now if time.to_i == 0
+        time = Fluent::Engine.now if time.to_i == 0
         record[@source_hostname_key] = peeraddr[2] if @source_hostname_key
         router.emit(tag, time, record)
         option = msg[3]
@@ -387,7 +334,7 @@ module Fluent
     end
 
     def check_and_skip_invalid_event(tag, es, peeraddr)
-      new_es = MultiEventStream.new
+      new_es = Fluent::MultiEventStream.new
       es.each { |time, record|
         if invalid_event?(tag, time, record)
           log.warn "skip invalid event:", source: source_message(peeraddr), tag: tag, time: time, record: record
@@ -399,7 +346,7 @@ module Fluent
     end
 
     def add_source_host(es, host)
-      new_es = MultiEventStream.new
+      new_es = Fluent::MultiEventStream.new
       es.each { |time, record|
         record[@source_hostname_key] = host
         new_es.add(time, record)
@@ -430,33 +377,6 @@ module Fluent
       ['HELO', {'nonce' => nonce, 'auth' => (@security ? user_auth_salt : ''), 'keepalive' => !@deny_keepalive}]
     end
 
-    ##### Authentication Handshake
-    #
-    # 1. (client) connect to server
-    #   * Socket handshake, checks certificate and its significate (in client, if using SSL)
-    # 2. (server)
-    #   * check network/domain acl (if enabled)
-    #   * disconnect when failed
-    # 3. (server) send HELO
-    #   * ['HELO', options(hash)]
-    #   * options:
-    #     * nonce: string (required)
-    #     * auth: string or blank_string (string: authentication required, and its salt is this value)
-    # 4. (client) send PING
-    #   * ['PING', selfhostname, sharedkey_salt, sha512_hex(sharedkey_salt + selfhostname + nonce + sharedkey), username || '', sha512_hex(auth_salt + username + password) || '']
-    # 5. (server) check PING
-    #   * check sharedkey
-    #   * check username / password (if required)
-    #   * send PONG FAILURE if failed
-    #   * ['PONG', false, 'reason of authentication failure', '', '']
-    # 6. (server) send PONG
-    #   * ['PONG', bool(authentication result), 'reason if authentication failed', selfhostname, sha512_hex(salt + selfhostname + nonce + sharedkey)]
-    # 7. (client) check PONG
-    #   * check sharedkey
-    #   * disconnect when failed
-    # 8. connection established
-    #   * send data from client
-
     def check_ping(message, remote_addr, user_auth_salt, nonce)
       log.debug "checking ping"
       # ['PING', self_hostname, shared_key_salt, sha512_hex(shared_key_salt + self_hostname + nonce + shared_key), username || '', sha512_hex(auth_salt + username + password) || '']

data/lib/fluent/plugin/in_http.rb

@@ -14,28 +14,29 @@
 #    limitations under the License.
 #
 
+require 'fluent/plugin/input'
+require 'fluent/plugin/parser'
+require 'fluent/event'
+
+require 'http/parser'
+require 'webrick/httputils'
 require 'uri'
 require 'socket'
 require 'json'
 
-require 'cool.io'
-
-require 'fluent/input'
-require 'fluent/event'
-require 'fluent/process'
+module Fluent::Plugin
+  class InHttpParser < Parser
+    Fluent::Plugin.register_parser('in_http', self)
+    def parse(text)
+      # this plugin is dummy implementation not to raise error
+      yield nil, nil
+    end
+  end
 
-module Fluent
   class HttpInput < Input
-    Plugin.register_input('http', self)
-
-    include DetachMultiProcessMixin
-
-    require 'http/parser'
+    Fluent::Plugin.register_input('http', self)
 
-    def initialize
-      require 'webrick/httputils'
-      super
-    end
+    helpers :parser, :compat_parameters, :event_loop
 
     EMPTY_GIF_IMAGE = "GIF89a\u0001\u0000\u0001\u0000\x80\xFF\u0000\xFF\xFF\xFF\u0000\u0000\u0000,\u0000\u0000\u0000\u0000\u0001\u0000\u0001\u0000\u0000\u0002\u0002D\u0001\u0000;".force_encoding("UTF-8")
 
@@ -52,25 +53,36 @@ module Fluent
     config_param :add_http_headers, :bool, default: false
     desc 'Add REMOTE_ADDR header to the record.'
     config_param :add_remote_addr, :bool, default: false
-    desc 'The format of the HTTP body.'
-    config_param :format, :string, default: 'default'
     config_param :blocking_timeout, :time, default: 0.5
     desc 'Set a white list of domains that can do CORS (Cross-Origin Resource Sharing)'
     config_param :cors_allow_origins, :array, default: nil
     desc 'Respond with empty gif image of 1x1 pixel.'
     config_param :respond_with_empty_img, :bool, default: false
 
+    config_section :parse do
+      config_set_default :@type, 'in_http'
+    end
+
+    EVENT_RECORD_PARAMETER = '_event_record'
+
     def configure(conf)
+      compat_parameters_convert(conf, :parser)
+
       super
 
-      m = if @format == 'default'
+      m = if @parser_configs.first['@type'] == 'in_http'
+            @parser_msgpack = parser_create(usage: 'parser_in_http_msgpack', type: 'msgpack')
+            @parser_msgpack.estimate_current_event = false
+            @parser_json = parser_create(usage: 'parser_in_http_json', type: 'json')
+            @parser_json.estimate_current_event = false
+            @format_name = 'default'
             method(:parse_params_default)
           else
-            @parser = Plugin.new_parser(@format)
-            @parser.configure(conf)
+            @parser = parser_create
+            @format_name = @parser_configs.first['@type']
             method(:parse_params_with_parser)
           end
-      (class << self; self; end).module_eval do
+      self.singleton_class.module_eval do
         define_method(:parse_params, m)
       end
     end
@@ -100,7 +112,11 @@ module Fluent
     end
 
     def start
-      log.debug "listening http on #{@bind}:#{@port}"
+      @_event_loop_run_timeout = @blocking_timeout
+
+      super
+
+      log.debug "listening http", bind: @bind, port: @port
 
       socket_manager_path = ENV['SERVERENGINE_SOCKETMANAGER_PATH']
       if Fluent.windows?
@@ -109,38 +125,24 @@ module Fluent
       client = ServerEngine::SocketManager::Client.new(socket_manager_path)
       lsock = client.listen_tcp(@bind, @port)
 
-      detach_multi_process do
-        super
-        @km = KeepaliveManager.new(@keepalive_timeout)
-        @lsock = Coolio::TCPServer.new(lsock, nil, Handler, @km, method(:on_request),
-                                       @body_size_limit, @format, log,
-                                       @cors_allow_origins)
-        @lsock.listen(@backlog) unless @backlog.nil?
-
-        @loop = Coolio::Loop.new
-        @loop.attach(@km)
-        @loop.attach(@lsock)
-
-        @thread = Thread.new(&method(:run))
-      end
+      @km = KeepaliveManager.new(@keepalive_timeout)
+      @lsock = Coolio::TCPServer.new(
+        lsock, nil, Handler, @km, method(:on_request),
+        @body_size_limit, @format_name, log,
+        @cors_allow_origins
+      )
+      @lsock.listen(@backlog) unless @backlog.nil?
+      event_loop_attach(@km)
+      event_loop_attach(@lsock)
+
+      @float_time_parser = Fluent::NumericTimeParser.new(:float)
     end
 
-    def shutdown
-      @loop.watchers.each {|w| w.detach }
-      @loop.stop
+    def close
       @lsock.close
-      @thread.join
-
       super
     end
 
-    def run
-      @loop.run(@blocking_timeout)
-    rescue
-      log.error "unexpected error", error: $!.to_s
-      log.error_backtrace
-    end
-
     def on_request(path_info, params)
       begin
         path = path_info[1..-1]  # remove /
@@ -170,9 +172,9 @@ module Fluent
         end
         time = if param_time = params['time']
                  param_time = param_time.to_f
-                 param_time.zero? ? Engine.now : Fluent::EventTime.from_time(Time.at(param_time))
+                 param_time.zero? ? Fluent::Engine.now : @float_time_parser.parse(param_time)
                else
-                 record_time.nil? ? Engine.now : record_time
+                 record_time.nil? ? Fluent::Engine.now : record_time
                end
       rescue
         return ["400 Bad Request", {'Content-Type'=>'text/plain'}, "400 Bad Request\n#{$!}\n"]
@@ -182,7 +184,7 @@ module Fluent
       begin
         # Support batched requests
         if record.is_a?(Array)
-          mes = MultiEventStream.new
+          mes = Fluent::MultiEventStream.new
           record.each do |single_record|
             if @add_http_headers
               params.each_pair { |k,v|
@@ -215,22 +217,23 @@ module Fluent
     private
 
     def parse_params_default(params)
-      record = if msgpack = params['msgpack']
-                 Engine.msgpack_factory.unpacker.feed(msgpack).read
-               elsif js = params['json']
-                 JSON.parse(js)
-               else
-                 raise "'json' or 'msgpack' parameter is required"
-               end
-      return nil, record
+      if msgpack = params['msgpack']
+        @parser_msgpack.parse(msgpack) do |_time, record|
+          return nil, record
+        end
+      elsif js = params['json']
+        @parser_json.parse(js) do |_time, record|
+          return nil, record
+        end
+      else
+        raise "'json' or 'msgpack' parameter is required"
+      end
     end
 
-    EVENT_RECORD_PARAMETER = '_event_record'
-
     def parse_params_with_parser(params)
       if content = params[EVENT_RECORD_PARAMETER]
         @parser.parse(content) { |time, record|
-          raise "Received event is not #{@format}: #{content}" if record.nil?
+          raise "Received event is not #{@format_name}: #{content}" if record.nil?
           return time, record
         }
       else
@@ -241,13 +244,13 @@ module Fluent
     class Handler < Coolio::Socket
       attr_reader :content_type
 
-      def initialize(io, km, callback, body_size_limit, format, log, cors_allow_origins)
+      def initialize(io, km, callback, body_size_limit, format_name, log, cors_allow_origins)
         super(io)
         @km = km
         @callback = callback
         @body_size_limit = body_size_limit
         @next_close = false
-        @format = format
+        @format_name = format_name
         @log = log
         @cors_allow_origins = cors_allow_origins
         @idle = 0
@@ -355,7 +358,7 @@ module Fluent
         uri = URI.parse(@parser.request_url)
         params = WEBrick::HTTPUtils.parse_query(uri.query)
 
-        if @format != 'default'
+        if @format_name != 'default'
           params[EVENT_RECORD_PARAMETER] = @body
         elsif @content_type =~ /^application\/x-www-form-urlencoded/
           params.update WEBrick::HTTPUtils.parse_query(@body)