fluentd 0.14.7-x64-mingw32 → 0.14.10-x64-mingw32

data/lib/fluent/plugin/in_forward.rb

@@ -14,34 +14,36 @@
 #    limitations under the License.
 #
 
-require 'fcntl'
 
-require 'cool.io'
+require 'fluent/plugin/input'
+require 'fluent/msgpack_factory'
 require 'yajl'
+require 'digest'
 
-require 'fluent/input'
-
-module Fluent
+module Fluent::Plugin
   class ForwardInput < Input
-    Plugin.register_input('forward', self)
+    Fluent::Plugin.register_input('forward', self)
 
-    LISTEN_PORT = 24224
+    # See the wiki page below for protocol specification
+    # https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1
 
-    def initialize
-      super
-      require 'fluent/plugin/socket_util'
-    end
+    helpers :server
+
+    LISTEN_PORT = 24224
 
     desc 'The port to listen to.'
     config_param :port, :integer, default: LISTEN_PORT
     desc 'The bind address to listen to.'
     config_param :bind, :string, default: '0.0.0.0'
+
     config_param :backlog, :integer, default: nil
     # SO_LINGER 0 to send RST rather than FIN to avoid lots of connections sitting in TIME_WAIT at src
     desc 'The timeout time used to set linger option.'
     config_param :linger_timeout, :integer, default: 0
     # This option is for Cool.io's loop wait timeout to avoid loop stuck at shutdown. Almost users don't need to change this value.
     config_param :blocking_timeout, :time, default: 0.5
+    desc 'Try to resolve hostname from IP addresses or not.'
+    config_param :resolve_hostname, :bool, default: nil
     desc 'Connections will be disconnected right after receiving first message if this value is true.'
     config_param :deny_keepalive, :bool, default: false
 
@@ -88,6 +90,15 @@ module Fluent
     def configure(conf)
       super
 
+      if @source_hostname_key
+        # TODO: add test
+        if @resolve_hostname.nil?
+          @resolve_hostname = true
+        elsif !@resolve_hostname # user specifies "false" in config
+          raise Fluent::ConfigError, "resolve_hostname must be true with source_hostname_key"
+        end
+      end
+
       if @security
         if @security.user_auth && @security.users.empty?
           raise Fluent::ConfigError, "<user> sections required if user_auth enabled"
@@ -125,80 +136,38 @@ module Fluent
             })
         end
       end
+      @lsock = @usock = nil
     end
 
+    HEARTBEAT_UDP_PAYLOAD = "\0"
+
     def start
       super
 
-      @loop = Coolio::Loop.new
-
-      socket_manager_path = ENV['SERVERENGINE_SOCKETMANAGER_PATH']
-      if Fluent.windows?
-        socket_manager_path = socket_manager_path.to_i
+      server_create_connection(
+        :in_forward_server, @port,
+        bind: @bind,
+        shared: false,
+        resolve_name: @resolve_hostname,
+        linger_timeout: @linger_timeout,
+        backlog: @backlog,
+        &method(:handle_connection)
+      )
+
+      server_create(:in_forward_server_udp_heartbeat, @port, shared: false, proto: :udp, bind: @bind, resolve_name: @resolve_hostname, max_bytes: 128) do |data, sock|
+        log.trace "heartbeat udp data arrived", host: sock.remote_host, port: sock.remote_port, data: data
+        begin
+          sock.write HEARTBEAT_UDP_PAYLOAD
+        rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR
+          log.trace "error while heartbeat response", host: sock.remote_host, error: e
+        end
       end
-      client = ServerEngine::SocketManager::Client.new(socket_manager_path)
-
-      @lsock = listen(client)
-      @loop.attach(@lsock)
-
-      @usock = client.listen_udp(@bind, @port)
-      @usock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK)
-      @hbr = HeartbeatRequestHandler.new(@usock, method(:on_heartbeat_request))
-      @loop.attach(@hbr)
-
-      @thread = Thread.new(&method(:run))
-    end
-
-    def shutdown
-      # In test cases it occasionally appeared that when detaching a watcher, another watcher is also detached.
-      # In the case in the iteration of watchers, a watcher that has been already detached is intended to be detached
-      # and therfore RuntimeError occurs saying that it is not attached to a loop.
-      # It occurs only when testing for sending responses to ForwardOutput.
-      # Sending responses needs to write the socket that is previously used only to read
-      # and a handler has 2 watchers that is used to read and to write.
-      # This problem occurs possibly because those watchers are thought to be related to each other
-      # and when detaching one of them the other is also detached for some reasons.
-      # As a workaround, check if watchers are attached before detaching them.
-      @loop.watchers.each {|w| w.detach if w.attached? }
-      @loop.stop
-      @usock.close
-      @thread.join
-      @lsock.close
-
-      super
-    end
-
-    def listen(client)
-      log.info "listening fluent socket on #{@bind}:#{@port}"
-      sock = client.listen_tcp(@bind, @port)
-      s = Coolio::TCPServer.new(sock, nil, Handler, @linger_timeout, log, method(:handle_connection))
-      s.listen(@backlog) unless @backlog.nil?
-      s
-    end
-
-    #config_param :path, :string, :default => DEFAULT_SOCKET_PATH
-    #def listen
-    #  if File.exist?(@path)
-    #    File.unlink(@path)
-    #  end
-    #  FileUtils.mkdir_p File.dirname(@path)
-    #  log.debug "listening fluent socket on #{@path}"
-    #  Coolio::UNIXServer.new(@path, Handler, method(:on_message))
-    #end
-
-    def run
-      @loop.run(@blocking_timeout)
-    rescue => e
-      log.error "unexpected error", error: e
-      log.error_backtrace
     end
 
-    private
-
     def handle_connection(conn)
       send_data = ->(serializer, data){ conn.write serializer.call(data) }
 
-      log.trace "connected fluent socket", address: conn.remote_addr, port: conn.remote_port
+      log.trace "connected fluent socket", addr: conn.remote_addr, port: conn.remote_port
       state = :established
       nonce = nil
       user_auth_salt = nil
@@ -212,7 +181,7 @@ module Fluent
         state = :pingpong
       end
 
-      log.trace "accepted fluent socket", address: conn.remote_addr, port: conn.remote_port
+      log.trace "accepted fluent socket", addr: conn.remote_addr, port: conn.remote_port
 
       read_messages(conn) do |msg, chunk_size, serializer|
         case state
@@ -228,15 +197,11 @@ module Fluent
           log.debug "connection established", address: conn.remote_addr, port: conn.remote_port
           state = :established
         when :established
-          options = on_message(msg, chunk_size, conn.remote_addr)
+          options = on_message(msg, chunk_size, conn.remote_host)
           if options && r = response(options)
-            send_data.call(serializer, r)
             log.trace "sent response to fluent socket", address: conn.remote_addr, response: r
-            if @deny_keepalive
-              conn.on_write_complete do
-                conn.close
-              end
-            end
+            conn.on_write_complete{ conn.close } if @deny_keepalive
+            send_data.call(serializer, r)
           else
             if @deny_keepalive
               conn.close
@@ -252,7 +217,7 @@ module Fluent
       feeder = nil
       serializer = nil
       bytes = 0
-      conn.on_data do |data|
+      conn.data do |data|
         # only for first call of callback
         unless feeder
           first = data[0]
@@ -288,30 +253,7 @@ module Fluent
       nil
     end
 
-    # message Entry {
-    #   1: long time
-    #   2: object record
-    # }
-    #
-    # message Forward {
-    #   1: string tag
-    #   2: list<Entry> entries
-    #   3: object option (optional)
-    # }
-    #
-    # message PackedForward {
-    #   1: string tag
-    #   2: raw entries  # msgpack stream of Entry
-    #   3: object option (optional)
-    # }
-    #
-    # message Message {
-    #   1: string tag
-    #   2: long? time
-    #   3: object record
-    #   4: object option (optional)
-    # }
-    def on_message(msg, chunk_size, peeraddr)
+    def on_message(msg, chunk_size, remote_host)
       if msg.nil?
         # for future TCP heartbeat_request
         return
@@ -319,7 +261,7 @@ module Fluent
 
       # TODO: raise an exception if broken chunk is generated by recoverable situation
       unless msg.is_a?(Array)
-        log.warn "incoming chunk is broken:", source: source_message(peeraddr), msg: msg
+        log.warn "incoming chunk is broken:", host: remote_host, msg: msg
         return
       end
 
@@ -327,10 +269,10 @@ module Fluent
       entries = msg[1]
 
       if @chunk_size_limit && (chunk_size > @chunk_size_limit)
-        log.warn "Input chunk size is larger than 'chunk_size_limit', dropped:", tag: tag, source: source_message(peeraddr), limit: @chunk_size_limit, size: chunk_size
+        log.warn "Input chunk size is larger than 'chunk_size_limit', dropped:", tag: tag, host: remote_host, limit: @chunk_size_limit, size: chunk_size
         return
       elsif @chunk_size_warn_limit && (chunk_size > @chunk_size_warn_limit)
-        log.warn "Input chunk size is larger than 'chunk_size_warn_limit':", tag: tag, source: source_message(peeraddr), limit: @chunk_size_warn_limit, size: chunk_size
+        log.warn "Input chunk size is larger than 'chunk_size_warn_limit':", tag: tag, host: remote_host, limit: @chunk_size_warn_limit, size: chunk_size
       end
 
       case entries
@@ -338,28 +280,28 @@ module Fluent
         # PackedForward
         option = msg[2]
         size = (option && option['size']) || 0
-        es_class = (option && option['compressed'] == 'gzip') ? CompressedMessagePackEventStream : MessagePackEventStream
+        es_class = (option && option['compressed'] == 'gzip') ? Fluent::CompressedMessagePackEventStream : Fluent::MessagePackEventStream
         es = es_class.new(entries, nil, size.to_i)
-        es = check_and_skip_invalid_event(tag, es, peeraddr) if @skip_invalid_event
-        es = add_source_host(es, peeraddr[2]) if @source_hostname_key
+        es = check_and_skip_invalid_event(tag, es, remote_host) if @skip_invalid_event
+        es = add_source_host(es, remote_host) if @source_hostname_key
         router.emit_stream(tag, es)
 
       when Array
         # Forward
         es = if @skip_invalid_event
-               check_and_skip_invalid_event(tag, entries, peeraddr)
+               check_and_skip_invalid_event(tag, entries, remote_host)
              else
-               es = MultiEventStream.new
+               es = Fluent::MultiEventStream.new
                entries.each { |e|
                  record = e[1]
                  next if record.nil?
                  time = e[0]
-                 time = (now ||= Engine.now) if time.to_i == 0
+                 time = Fluent::Engine.now if time.nil? || time.to_i == 0 # `to_i == 0` for empty EventTime
                  es.add(time, record)
                }
                es
              end
-        es = add_source_host(es, peeraddr[2]) if @source_hostname_key
+        es = add_source_host(es, remote_host) if @source_hostname_key
         router.emit_stream(tag, es)
         option = msg[2]
 
@@ -368,12 +310,12 @@ module Fluent
         time = msg[1]
         record = msg[2]
         if @skip_invalid_event && invalid_event?(tag, time, record)
-          log.warn "got invalid event and drop it:", source: source_message(peeraddr), tag: tag, time: time, record: record
+          log.warn "got invalid event and drop it:", host: remote_host, tag: tag, time: time, record: record
           return msg[3] # retry never succeeded so return ack and drop incoming event.
         end
         return if record.nil?
-        time = Engine.now if time.to_i == 0
-        record[@source_hostname_key] = peeraddr[2] if @source_hostname_key
+        time = Fluent::Engine.now if time.to_i == 0
+        record[@source_hostname_key] = remote_host if @source_hostname_key
         router.emit(tag, time, record)
         option = msg[3]
       end
@@ -386,11 +328,11 @@ module Fluent
       !((time.is_a?(Integer) || time.is_a?(::Fluent::EventTime)) && record.is_a?(Hash) && tag.is_a?(String))
     end
 
-    def check_and_skip_invalid_event(tag, es, peeraddr)
-      new_es = MultiEventStream.new
+    def check_and_skip_invalid_event(tag, es, remote_host)
+      new_es = Fluent::MultiEventStream.new
       es.each { |time, record|
         if invalid_event?(tag, time, record)
-          log.warn "skip invalid event:", source: source_message(peeraddr), tag: tag, time: time, record: record
+          log.warn "skip invalid event:", host: remote_host, tag: tag, time: time, record: record
           next
         end
         new_es.add(time, record)
@@ -399,7 +341,7 @@ module Fluent
     end
 
     def add_source_host(es, host)
-      new_es = MultiEventStream.new
+      new_es = Fluent::MultiEventStream.new
       es.each { |time, record|
         record[@source_hostname_key] = host
         new_es.add(time, record)
@@ -407,11 +349,6 @@ module Fluent
       new_es
     end
 
-    def source_message(peeraddr)
-      _, port, host, addr = peeraddr
-      "host: #{host}, addr: #{addr}, port: #{port}"
-    end
-
     def select_authenticate_users(node, username)
       if node.nil? || node[:users].empty?
         @security.users.select{|u| u.username == username}
@@ -430,33 +367,6 @@ module Fluent
       ['HELO', {'nonce' => nonce, 'auth' => (@security ? user_auth_salt : ''), 'keepalive' => !@deny_keepalive}]
     end
 
-    ##### Authentication Handshake
-    #
-    # 1. (client) connect to server
-    #   * Socket handshake, checks certificate and its significate (in client, if using SSL)
-    # 2. (server)
-    #   * check network/domain acl (if enabled)
-    #   * disconnect when failed
-    # 3. (server) send HELO
-    #   * ['HELO', options(hash)]
-    #   * options:
-    #     * nonce: string (required)
-    #     * auth: string or blank_string (string: authentication required, and its salt is this value)
-    # 4. (client) send PING
-    #   * ['PING', selfhostname, sharedkey_salt, sha512_hex(sharedkey_salt + selfhostname + nonce + sharedkey), username || '', sha512_hex(auth_salt + username + password) || '']
-    # 5. (server) check PING
-    #   * check sharedkey
-    #   * check username / password (if required)
-    #   * send PONG FAILURE if failed
-    #   * ['PONG', false, 'reason of authentication failure', '', '']
-    # 6. (server) send PONG
-    #   * ['PONG', bool(authentication result), 'reason if authentication failed', selfhostname, sha512_hex(salt + selfhostname + nonce + sharedkey)]
-    # 7. (client) check PONG
-    #   * check sharedkey
-    #   * disconnect when failed
-    # 8. connection established
-    #   * send data from client
-
     def check_ping(message, remote_addr, user_auth_salt, nonce)
       log.debug "checking ping"
       # ['PING', self_hostname, shared_key_salt, sha512_hex(shared_key_salt + self_hostname + nonce + shared_key), username || '', sha512_hex(auth_salt + username + password) || '']
@@ -504,114 +414,5 @@ module Fluent
       shared_key_digest_hex = Digest::SHA512.new.update(reason_or_salt).update(@security.self_hostname).update(nonce).update(shared_key).hexdigest
       ['PONG', true, '', @security.self_hostname, shared_key_digest_hex]
     end
-
-    class Handler < Coolio::Socket
-      attr_reader :protocol, :remote_port, :remote_addr, :remote_host
-
-      PEERADDR_FAILED = ["?", "?", "name resolusion failed", "?"]
-
-      def initialize(io, linger_timeout, log, on_connect_callback)
-        super(io)
-
-        @peeraddr = nil
-        if io.is_a?(TCPSocket) # for unix domain socket support in the future
-          @peeraddr = (io.peeraddr rescue PEERADDR_FAILED)
-          opt = [1, linger_timeout].pack('I!I!')  # { int l_onoff; int l_linger; }
-          io.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
-        end
-
-        ### TODO: disabling name rev resolv
-        proto, port, host, addr = ( io.peeraddr rescue PEERADDR_FAILED )
-        if addr == '?'
-          port, addr = *Socket.unpack_sockaddr_in(io.getpeername) rescue nil
-        end
-        @protocol = proto
-        @remote_port = port
-        @remote_addr = addr
-        @remote_host = host
-        @writing = false
-        @closing = false
-        @mutex = Mutex.new
-
-        @chunk_counter = 0
-        @on_connect_callback = on_connect_callback
-        @log = log
-        @log.trace {
-          begin
-            remote_port, remote_addr = *Socket.unpack_sockaddr_in(@_io.getpeername)
-          rescue
-            remote_port = nil
-            remote_addr = nil
-          end
-          [ "accepted fluent socket", {address: remote_addr, port: remote_port, instance: self.object_id} ]
-        }
-      end
-
-      def on_connect
-        @on_connect_callback.call(self)
-      end
-
-      # API to register callback for data arrival
-      def on_data(&callback)
-        @on_read_callback = callback
-      end
-
-      def on_read(data)
-        @on_read_callback.call(data)
-      rescue => e
-        @log.error "unexpected error on reading data from client", address: @remote_addr, error: e
-        @log.error_backtrace
-        close
-      end
-
-      def on_write_complete
-        closing = @mutex.synchronize {
-          @writing = false
-          @closing
-        }
-        if closing
-          close
-        end
-      end
-
-      def close
-        writing = @mutex.synchronize {
-          @closing = true
-          @writing
-        }
-        unless writing
-          super
-        end
-      end
-    end
-
-    class HeartbeatRequestHandler < Coolio::IO
-      def initialize(io, callback)
-        super(io)
-        @io = io
-        @callback = callback
-      end
-
-      def on_readable
-        begin
-          msg, addr = @io.recvfrom(1024)
-        rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR
-          return
-        end
-        host = addr[3]
-        port = addr[1]
-        @callback.call(host, port, msg)
-      rescue
-        # TODO log?
-      end
-    end
-
-    def on_heartbeat_request(host, port, msg)
-      #log.trace "heartbeat request from #{host}:#{port}"
-      begin
-        @usock.send "\0", 0, host, port
-      rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR
-      end
-    end
   end
 end