fluentd 0.14.9 → 0.14.10

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -18,8 +18,7 @@ require 'json'
 require 'webrick'
 require 'cgi'
 
-require 'cool.io'
-
+require 'fluent/config/types'
 require 'fluent/plugin/input'
 require 'fluent/plugin/output'
 require 'fluent/plugin/multi_output'
@@ -35,6 +34,7 @@ module Fluent::Plugin
     config_param :port, :integer, default: 24220
     config_param :tag, :string, default: nil
     config_param :emit_interval, :time, default: 60
+    config_param :include_config, :bool, default: true
 
     class MonitorServlet < WEBrick::HTTPServlet::AbstractServlet
       def initialize(server, agent)
@@ -78,12 +78,16 @@ module Fluent::Plugin
 
         # if ?debug=1 is set, set :with_debug_info for get_monitor_info
         # and :pretty_json for render_json_error
-        opts = {}
+        opts = {with_config: @agent.include_config}
         if s = qs['debug'] and s[0]
           opts[:with_debug_info] = true
           opts[:pretty_json] = true
         end
 
+        if with_config = get_search_parameter(qs, 'with_config'.freeze)
+          opts[:with_config] = Fluent::Config.bool_value(with_config)
+        end
+
         if tag = get_search_parameter(qs, 'tag'.freeze)
           # ?tag= to search an output plugin by match pattern
           if obj = @agent.plugin_info_by_tag(tag, opts)
@@ -329,7 +333,7 @@ module Fluent::Plugin
       obj['plugin_id'] = pe.plugin_id
       obj['plugin_category'] = plugin_category(pe)
       obj['type'] = pe.config['@type']
-      obj['config'] = pe.config if !opts.has_key?(:with_config) || opts[:with_config]
+      obj['config'] = pe.config if opts[:with_config]
 
       # run MONITOR_INFO in plugins' instance context and store the info to obj
       MONITOR_INFO.each_pair {|key,code|

data/lib/fluent/plugin/in_syslog.rb

@@ -85,6 +85,10 @@ module Fluent::Plugin
     config_param :include_source_host, :bool, default: false
     desc 'Specify key of source host when include_source_host is true.'
     config_param :source_host_key, :string, default: 'source_host'.freeze
+    desc 'The field name of the priority.'
+    config_param :priority_key, :string, default: nil
+    desc 'The field name of the facility.'
+    config_param :facility_key, :string, default: nil
     config_param :blocking_timeout, :time, default: 0.5
     config_param :message_length_limit, :size, default: 2048
 
@@ -141,8 +145,15 @@ module Fluent::Plugin
         end
 
         pri ||= record.delete('pri')
+        facility = FACILITY_MAP[pri >> 3]
+        priority = PRIORITY_MAP[pri & 0b111]
+
+        record[@priority_key] = priority if @priority_key
+        record[@facility_key] = facility if @facility_key
         record[@source_host_key] = addr[2] if @include_source_host
-        emit(pri, time, record)
+
+        tag = "#{@tag}.#{facility}.#{priority}"
+        emit(tag, time, record)
       end
     rescue => e
       log.error "invalid input", data: data, error: e
@@ -168,12 +179,7 @@ module Fluent::Plugin
       end
     end
 
-    def emit(pri, time, record)
-      facility = FACILITY_MAP[pri >> 3]
-      priority = PRIORITY_MAP[pri & 0b111]
-
-      tag = "#{@tag}.#{facility}.#{priority}"
-
+    def emit(tag, time, record)
       router.emit(tag, time, record)
     rescue => e
       log.error "syslog failed to emit", error: e, tag: tag, record: Yajl.dump(record)

data/lib/fluent/plugin/in_tail.rb

@@ -164,6 +164,12 @@ module Fluent::Plugin
       super
     end
 
+    def close
+      super
+      # close file handles after all threads stopped (in #close of thread plugin helper)
+      close_watcher_handles
+    end
+
     def expand_paths
       date = Time.now
       paths = []
@@ -234,18 +240,27 @@ module Fluent::Plugin
 
     def stop_watchers(paths, immediate = false, unwatched = false)
       paths.each { |path|
-        tw = @tails.delete(path)
+        tw = @tails[path]
         if tw
           tw.unwatched = unwatched
           if immediate
-            close_watcher(tw, false)
+            detach_watcher(tw, false)
           else
-            close_watcher_after_rotate_wait(tw)
+            detach_watcher_after_rotate_wait(tw)
           end
         end
       }
     end
 
+    def close_watcher_handles
+      @tails.keys.each do |path|
+        tw = @tails.delete(path)
+        if tw
+          tw.close
+        end
+      end
+    end
+
     # refresh_watchers calls @tails.keys so we don't use stop_watcher -> start_watcher sequence for safety.
     def update_watcher(path, pe)
       if @pf
@@ -256,24 +271,25 @@ module Fluent::Plugin
       end
       rotated_tw = @tails[path]
       @tails[path] = setup_watcher(path, pe)
-      close_watcher_after_rotate_wait(rotated_tw) if rotated_tw
+      detach_watcher_after_rotate_wait(rotated_tw) if rotated_tw
     end
 
     # TailWatcher#close is called by another thread at shutdown phase.
     # It causes 'can't modify string; temporarily locked' error in IOHandler
     # so adding close_io argument to avoid this problem.
     # At shutdown, IOHandler's io will be released automatically after detached the event loop
-    def close_watcher(tw, close_io = true)
-      tw.close(close_io)
+    def detach_watcher(tw, close_io = true)
+      tw.detach
+      tw.close if close_io
       flush_buffer(tw)
       if tw.unwatched && @pf
         @pf[tw.path].update_pos(PositionFile::UNWATCHED_POSITION)
       end
     end
 
-    def close_watcher_after_rotate_wait(tw)
+    def detach_watcher_after_rotate_wait(tw)
       timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do
-        close_watcher(tw)
+        detach_watcher(tw)
       end
     end
 
@@ -435,14 +451,13 @@ module Fluent::Plugin
       def detach
         @timer_trigger.detach if @enable_watch_timer && @timer_trigger.attached?
         @stat_trigger.detach if @stat_trigger.attached?
+        @io_handler.on_notify if @io_handler
       end
 
-      def close(close_io = true)
-        if close_io && @io_handler
-          @io_handler.on_notify
+      def close
+        if @io_handler
           @io_handler.close
         end
-        detach
       end
 
       def on_notify
@@ -570,8 +585,8 @@ module Fluent::Plugin
                   else
                     @buffer << @io.readpartial(2048, @iobuf)
                   end
-                  while line = @buffer.slice!(/.*?\n/m)
-                    @lines << line
+                  while idx = @buffer.index("\n".freeze)
+                    @lines << @buffer.slice!(0, idx + 1)
                   end
                   if @lines.size >= @read_lines_limit
                     # not to use too much memory in case the file is very large

data/lib/fluent/plugin/in_tcp.rb

@@ -14,28 +14,68 @@
 #    limitations under the License.
 #
 
-require 'cool.io'
+require 'fluent/plugin/input'
 
-require 'fluent/plugin/socket_util'
+module Fluent::Plugin
+  class TcpInput < Input
+    Fluent::Plugin.register_input('tcp', self)
 
-module Fluent
-  class TcpInput < SocketUtil::BaseInput
-    Plugin.register_input('tcp', self)
+    helpers :server, :parser, :extract, :compat_parameters
+
+    desc 'Tag of output events.'
+    config_param :tag, :string
+    desc 'The port to listen to.'
+    config_param :port, :integer, default: 5170
+    desc 'The bind address to listen to.'
+    config_param :bind, :string, default: '0.0.0.0'
+
+    desc "The field name of the client's hostname."
+    config_param :source_host_key, :string, default: nil, deprecated: "use source_hostname_key instead."
+    desc "The field name of the client's hostname."
+    config_param :source_hostname_key, :string, default: nil
+
+    config_param :blocking_timeout, :time, default: 0.5
 
-    config_set_default :port, 5170
     desc 'The payload is read up to this character.'
     config_param :delimiter, :string, default: "\n" # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
 
-    def listen(callback)
-      log.info "listening tcp socket on #{@bind}:#{@port}"
+    def configure(conf)
+      compat_parameters_convert(conf, :parser)
+      super
+      @_event_loop_blocking_timeout = @blocking_timeout
+      @source_hostname_key ||= @source_host_key if @source_host_key
+
+      @parser = parser_create
+    end
+
+    def start
+      super
+
+      @buffer = ''
+      server_create(:in_tcp_server, @port, proto: :tcp, bind: @bind) do |data, conn|
+        @buffer << data
+        begin
+          pos = 0
+          while i = @buffer.index(@delimiter, pos)
+            msg = @buffer[pos...i]
+            pos = i + @delimiter.length
+
+            @parser.parse(msg) do |time, record|
+              unless time && record
+                log.warn "pattern not match", message: msg
+                next
+              end
 
-      socket_manager_path = ENV['SERVERENGINE_SOCKETMANAGER_PATH']
-      if Fluent.windows?
-        socket_manager_path = socket_manager_path.to_i
+              tag = extract_tag_from_record(record)
+              tag ||= @tag
+              time ||= extract_time_from_record(record) || Fluent::EventTime.now
+              record[@source_hostname_key] = conn.remote_host if @source_hostname_key
+              router.emit(tag, time, record)
+            end
+          end
+          @buffer.slice!(0, pos) if pos > 0
+        end
       end
-      client = ServerEngine::SocketManager::Client.new(socket_manager_path)
-      lsock = client.listen_tcp(@bind, @port)
-      Coolio::TCPServer.new(lsock, nil, SocketUtil::TcpHandler, log, @delimiter, callback)
     end
   end
 end

data/lib/fluent/plugin/in_udp.rb

@@ -14,24 +14,60 @@
 #    limitations under the License.
 #
 
-require 'fluent/plugin/socket_util'
+require 'fluent/plugin/input'
 
-module Fluent
-  class UdpInput < SocketUtil::BaseInput
-    Plugin.register_input('udp', self)
+module Fluent::Plugin
+  class UdpInput < Input
+    Fluent::Plugin.register_input('udp', self)
+
+    helpers :server, :parser, :extract, :compat_parameters
+
+    desc 'Tag of output events.'
+    config_param :tag, :string
+    desc 'The port to listen to.'
+    config_param :port, :integer, default: 5160
+    desc 'The bind address to listen to.'
+    config_param :bind, :string, default: '0.0.0.0'
+
+    desc "The field name of the client's hostname."
+    config_param :source_host_key, :string, default: nil, deprecated: "use source_hostname_key instead."
+    desc "The field name of the client's hostname."
+    config_param :source_hostname_key, :string, default: nil
 
-    config_set_default :port, 5160
     config_param :body_size_limit, :size, default: 4096
 
-    def listen(callback)
-      log.info "listening udp socket on #{@bind}:#{@port}"
-      socket_manager_path = ENV['SERVERENGINE_SOCKETMANAGER_PATH']
-      if Fluent.windows?
-        socket_manager_path = socket_manager_path.to_i
+    config_param :blocking_timeout, :time, default: 0.5
+
+    def configure(conf)
+      compat_parameters_convert(conf, :parser)
+      super
+      @_event_loop_blocking_timeout = @blocking_timeout
+      @source_hostname_key ||= @source_host_key if @source_host_key
+
+      @parser = parser_create
+    end
+
+    def start
+      super
+
+      log.info "listening udp socket", bind: @bind, port: @port
+      server_create(:in_udp_server, @port, proto: :udp, bind: @bind, max_bytes: @body_size_limit) do |data, sock|
+        data.chomp!
+        begin
+          @parser.parse(data) do |time, record|
+            unless time && record
+              log.warn "pattern not match", data: data
+              next
+            end
+
+            tag = extract_tag_from_record(record)
+            tag ||= @tag
+            time ||= extract_time_from_record(record) || Fluent::EventTime.now
+            record[@source_hostname_key] = sock.remote_host if @source_hostname_key
+            router.emit(tag, time, record)
+          end
+        end
       end
-      client = ServerEngine::SocketManager::Client.new(socket_manager_path)
-      @usock = client.listen_udp(@bind, @port)
-      SocketUtil::UdpHandler.new(@usock, log, @body_size_limit, callback)
     end
   end
 end

data/lib/fluent/plugin/out_file.rb

@@ -113,25 +113,41 @@ module Fluent::Plugin
       end
 
       path_suffix = @add_path_suffix ? @path_suffix : ''
-      @path_template = generate_path_template(@path, @buffer_config.timekey, @append, @compress_method, path_suffix: path_suffix, time_slice_format: configured_time_slice_format)
-
-      placeholder_validate!(:path, @path_template)
-
-      max_tag_index = get_placeholders_tag(@path_template).max || 1
-      max_tag_index = 1 if max_tag_index < 1
-      dummy_tag = (['a'] * max_tag_index).join('.')
-      dummy_record_keys = get_placeholders_keys(@path_template) || ['message']
-      dummy_record = Hash[dummy_record_keys.zip(['data'] * dummy_record_keys.size)]
-
-      test_meta1 = metadata_for_test(dummy_tag, Fluent::Engine.now, dummy_record)
-      test_path = extract_placeholders(@path_template, test_meta1)
-      unless ::Fluent::FileUtil.writable_p?(test_path)
-        raise Fluent::ConfigError, "out_file: `#{test_path}` is not writable"
+      path_timekey = if @chunk_key_time
+                       @as_secondary ? @primary_instance.buffer_config.timekey : @buffer_config.timekey
+                     else
+                       nil
+                     end
+      @path_template = generate_path_template(@path, path_timekey, @append, @compress_method, path_suffix: path_suffix, time_slice_format: configured_time_slice_format)
+
+      if @as_secondary
+        # When this plugin is configured as secondary & primary plugin has tag key, but this plugin may not have it.
+        # Increment placeholder can make another output file per chunk tag/keys even if original path doesn't include it.
+        placeholder_validators(:path, @path_template).select{|v| v.type == :time }.each do |v|
+          v.validate!
+        end
+      else
+        placeholder_validate!(:path, @path_template)
+
+        max_tag_index = get_placeholders_tag(@path_template).max || 1
+        max_tag_index = 1 if max_tag_index < 1
+        dummy_tag = (['a'] * max_tag_index).join('.')
+        dummy_record_keys = get_placeholders_keys(@path_template) || ['message']
+        dummy_record = Hash[dummy_record_keys.zip(['data'] * dummy_record_keys.size)]
+
+        test_meta1 = metadata_for_test(dummy_tag, Fluent::Engine.now, dummy_record)
+        test_path = extract_placeholders(@path_template, test_meta1)
+        unless ::Fluent::FileUtil.writable_p?(test_path)
+          raise Fluent::ConfigError, "out_file: `#{test_path}` is not writable"
+        end
       end
 
       @formatter = formatter_create
 
       if @symlink_path && @buffer.respond_to?(:path)
+        if @as_secondary
+          raise Fluent::ConfigError, "symlink_path option is unavailable in <secondary>: consider to use secondary_file plugin"
+        end
         if Fluent.windows?
           log.warn "symlink_path is unavailable on Windows platform. disabled."
           @symlink_path = nil

data/lib/fluent/plugin/out_forward.rb

@@ -23,16 +23,21 @@ require 'fluent/compat/socket_util'
 module Fluent::Plugin
   class ForwardOutput < Output
     class Error < StandardError; end
-    class ResponseError < Error; end
+    class NoNodesAvailable < Error; end
     class ConnectionClosedError < Error; end
-    class ACKTimeoutError < Error; end
 
     Fluent::Plugin.register_output('forward', self)
 
+    helpers :socket, :server, :timer, :thread, :compat_parameters
+
     LISTEN_PORT = 24224
 
+    PROCESS_CLOCK_ID = Process::CLOCK_MONOTONIC_RAW rescue Process::CLOCK_MONOTONIC
+
     desc 'The timeout time when sending event logs.'
     config_param :send_timeout, :time, default: 60
+    # TODO: add linger_timeout, recv_timeout
+
     desc 'The transport protocol to use for heartbeats.(udp,tcp,none)'
     config_param :heartbeat_type, :enum, list: [:tcp, :udp, :none], default: :tcp
     desc 'The interval of the heartbeat packer.'
@@ -41,8 +46,6 @@ module Fluent::Plugin
     config_param :recover_wait, :time, default: 10
     desc 'The hard timeout used to detect server failure.'
     config_param :hard_timeout, :time, default: 60
-    desc 'Set TTL to expire DNS cache in seconds.'
-    config_param :expire_dns_cache, :time, default: nil  # 0 means disable cache
     desc 'The threshold parameter used to detect server faults.'
     config_param :phi_threshold, :integer, default: 16
     desc 'Use the "Phi accrual failure detector" to detect server failure.'
@@ -50,14 +53,20 @@ module Fluent::Plugin
 
     desc 'Change the protocol to at-least-once.'
     config_param :require_ack_response, :bool, default: false  # require in_forward to respond with ack
+
+    ## The reason of default value of :ack_response_timeout:
+    # Linux default tcp_syn_retries is 5 (in many environment)
+    # 3 + 6 + 12 + 24 + 48 + 96 -> 189 (sec)
     desc 'This option is used when require_ack_response is true.'
     config_param :ack_response_timeout, :time, default: 190
-    desc 'Reading data size from server'
-    config_param :read_length, :size, default: 512 # 512bytes
+
     desc 'The interval while reading data from server'
     config_param :read_interval_msec, :integer, default: 50 # 50ms
-    # Linux default tcp_syn_retries is 5 (in many environment)
-    # 3 + 6 + 12 + 24 + 48 + 96 -> 189 (sec)
+    desc 'Reading data size from server'
+    config_param :read_length, :size, default: 512 # 512bytes
+
+    desc 'Set TTL to expire DNS cache in seconds.'
+    config_param :expire_dns_cache, :time, default: nil  # 0 means disable cache
     desc 'Enable client-side DNS round robin.'
     config_param :dns_round_robin, :bool, default: false # heartbeat_type 'udp' is not available for this
 
@@ -107,10 +116,15 @@ module Fluent::Plugin
       @nodes = [] #=> [Node]
       @loop = nil
       @thread = nil
-      @finished = false
+
+      @usock = nil
+      @sock_ack_waiting = nil
+      @sock_ack_waiting_mutex = nil
     end
 
     def configure(conf)
+      compat_parameters_convert(conf, :buffer, default_chunk_key: 'tag')
+
       super
 
       unless @chunk_key_tag
@@ -153,79 +167,104 @@ module Fluent::Plugin
       raise Fluent::ConfigError, "ack_response_timeout must be a positive integer" if @ack_response_timeout < 1
     end
 
+    def prefer_delayed_commit
+      @require_ack_response
+    end
+
     def start
       super
 
+      # Output#start sets @delayed_commit_timeout by @buffer_config.delayed_commit_timeout
+      # But it should be overwritten by ack_response_timeout to rollback chunks after timeout
+      if @ack_response_timeout && @delayed_commit_timeout != @ack_response_timeout
+        log.info "delayed_commit_timeout is overwritten by ack_response_timeout"
+        @delayed_commit_timeout = @ack_response_timeout
+      end
+
       @rand_seed = Random.new.seed
       rebuild_weight_array
       @rr = 0
-      @usock = nil
 
       unless @heartbeat_type == :none
-        @loop = Coolio::Loop.new
-
         if @heartbeat_type == :udp
-          # assuming all hosts use udp
-          @usock = Fluent::Compat::SocketUtil.create_udp_socket(@nodes.first.host)
-          @usock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK)
-          @hb = HeartbeatHandler.new(@usock, method(:on_heartbeat))
-          @loop.attach(@hb)
+          @usock = socket_create_udp(@nodes.first.host, @nodes.first.port, nonblock: true)
+          server_create_udp(:out_forward_heartbeat_receiver, 0, socket: @usock, max_bytes: @read_length) do |data, sock|
+            sockaddr = Socket.pack_sockaddr_in(sock.remote_port, sock.remote_host)
+            on_heartbeat(sockaddr, data)
+          end
         end
+        timer_execute(:out_forward_heartbeat_request, @heartbeat_interval, &method(:on_timer))
+      end
 
-        @timer = HeartbeatRequestTimer.new(@heartbeat_interval, method(:on_timer))
-        @loop.attach(@timer)
-
-        @thread = Thread.new(&method(:run))
+      if @require_ack_response
+        @sock_ack_waiting_mutex = Mutex.new
+        @sock_ack_waiting = []
+        thread_create(:out_forward_receiving_ack, &method(:ack_reader))
       end
     end
 
-    def shutdown
-      @finished = true
-      if @loop
-        @loop.watchers.each {|w| w.detach }
-        # @loop.stop
-        @loop.stop rescue nil
-      end
-      @thread.join if @thread
+    def close
       @usock.close if @usock
-
       super
     end
 
-    def run
-      @loop.run if @loop
-    rescue
-      log.error "unexpected error", error: $!.to_s
-      log.error_backtrace
-    end
-
     def write(chunk)
       return if chunk.empty?
+      tag = chunk.metadata.tag
+      select_a_healthy_node{|node| node.send_data(tag, chunk) }
+    end
 
+    ACKWaitingSockInfo = Struct.new(:sock, :chunk_id, :node, :time, :timeout) do
+      def expired?(now)
+        time + timeout < now
+      end
+    end
+
+    def try_write(chunk)
+      if chunk.empty?
+        commit_write(chunk.unique_id)
+        return
+      end
       tag = chunk.metadata.tag
+      sock, node = select_a_healthy_node{|n| n.send_data(tag, chunk) }
+      chunk_id = Base64.encode64(chunk.unique_id)
+      current_time = Process.clock_gettime(PROCESS_CLOCK_ID)
+      info = ACKWaitingSockInfo.new(sock, chunk_id, node, current_time, @ack_response_timeout)
+      @sock_ack_waiting_mutex.synchronize do
+        @sock_ack_waiting << info
+      end
+    end
+
+    def select_a_healthy_node
       error = nil
 
       wlen = @weight_array.length
       wlen.times do
         @rr = (@rr + 1) % wlen
         node = @weight_array[@rr]
+        next unless node.available?
 
-        if node.available?
-          begin
-            node.send_data(tag, chunk)
-            return
-          rescue
-            # for load balancing during detecting crashed servers
-            error = $!  # use the latest error
-          end
+        begin
+          ret = yield node
+          return ret, node
+        rescue
+          # for load balancing during detecting crashed servers
+          error = $!  # use the latest error
         end
       end
 
-      if error
-        raise error
-      else
-        raise "no nodes are available"  # TODO message
-      end
+      raise error if error
+      raise NoNodesAvailable, "no nodes are available"
+    end
+
+    def create_transfer_socket(host, port, &block)
+      socket_create_tcp(
+        host, port,
+        linger_timeout: @send_timeout,
+        send_timeout: @send_timeout,
+        recv_timeout: @ack_response_timeout,
+        &block
+      )
     end
 
     # MessagePack FixArray length is 3
@@ -278,21 +317,7 @@ module Fluent::Plugin
       @weight_array = weight_array
     end
 
-    class HeartbeatRequestTimer < Coolio::TimerWatcher
-      def initialize(interval, callback)
-        super(interval, true)
-        @callback = callback
-      end
-
-      def on_timer
-        @callback.call
-      rescue
-        # TODO log?
-      end
-    end
-
     def on_timer
-      return if @finished
       @nodes.each {|n|
         if n.tick
           rebuild_weight_array
@@ -307,33 +332,86 @@ module Fluent::Plugin
       }
     end
 
-    class HeartbeatHandler < Coolio::IO
-      def initialize(io, callback)
-        super(io)
-        @io = io
-        @callback = callback
+    def on_heartbeat(sockaddr, msg)
+      if node = @nodes.find {|n| n.sockaddr == sockaddr }
+        # log.trace "heartbeat arrived", name: node.name, host: node.host, port: node.port
+        if node.heartbeat
+          rebuild_weight_array
+        end
       end
+    end
 
-      def on_readable
-        begin
-          msg, addr = @io.recvfrom(1024)
-        rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR
-          return
+    # return chunk id when succeeded for tests
+    def read_ack_from_sock(sock, unpacker)
+      begin
+        raw_data = sock.recv(@read_length)
+      rescue Errno::ECONNRESET
+        raw_data = ""
+      end
+      info = @sock_ack_waiting_mutex.synchronize{ @sock_ack_waiting.find{|i| i.sock == sock } }
+
+      # When connection is closed by remote host, socket is ready to read and #recv returns an empty string that means EOF.
+      # If this happens we assume the data wasn't delivered and retry it.
+      if raw_data.empty?
+        log.warn "destination node closed the connection. regard it as unavailable.", host: info.node.host, port: info.node.port
+        info.node.disable!
+        return nil
+      else
+        unpacker.feed(raw_data)
+        res = unpacker.read
+        if res['ack'] != info.chunk_id
+          # Some errors may have occured when ack and chunk id is different, so send the chunk again.
+          log.warn "ack in response and chunk id in sent data are different", chunk_id: info.chunk_id, ack: res['ack']
+          rollback_write(info.chunk_id)
+          return nil
         end
-        host = addr[3]
-        port = addr[1]
-        sockaddr = Socket.pack_sockaddr_in(port, host)
-        @callback.call(sockaddr, msg)
-      rescue
-        # TODO log?
+        return info.chunk_id
+      end
+    rescue => e
+      log.error "unexpected error while receiving ack message", error: e
+      log.error_backtrace
+    ensure
+      @sock_ack_waiting_mutex.synchronize do
+        @sock_ack_waiting.delete(info)
       end
     end
 
-    def on_heartbeat(sockaddr, msg)
-      if node = @nodes.find {|n| n.sockaddr == sockaddr }
-        #log.trace "heartbeat from '#{node.name}'", :host=>node.host, :port=>node.port
-        if node.heartbeat
-          rebuild_weight_array
+    def ack_reader
+      select_interval = if @delayed_commit_timeout > 3
+                          2
+                        else
+                          @delayed_commit_timeout / 2.0
+                        end
+
+      unpacker = Fluent::Engine.msgpack_unpacker
+
+      while thread_current_running?
+        now = Process.clock_gettime(PROCESS_CLOCK_ID)
+        sockets = []
+        @sock_ack_waiting_mutex.synchronize do
+          new_list = []
+          @sock_ack_waiting.each do |info|
+            if info.expired?(now)
+              # There are 2 types of cases when no response has been received from socket:
+              # (1) the node does not support sending responses
+              # (2) the node does support sending response but responses have not arrived for some reasons.
+              log.warn "no response from node. regard it as unavailable.", host: info.node.host, port: info.node.port
+              info.node.disable!
+              info.sock.close rescue nil
+              rollback_write(info.chunk_id)
+            else
+              sockets << info.sock
+              new_list << info
+            end
+          end
+          @sock_ack_waiting = new_list
+        end
+
+        readable_sockets, _, _ = IO.select(sockets, nil, nil, select_interval)
+        next unless readable_sockets
+
+        readable_sockets.each do |sock|
+          read_ack_from_sock(sock, unpacker)
         end
       end
     end
@@ -386,20 +464,6 @@ module Fluent::Plugin
         @standby
       end
 
-      def connect
-        TCPSocket.new(resolved_host, port)
-      end
-
-      def set_socket_options(sock)
-        opt = [1, @sender.send_timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
-        sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
-
-        opt = [@sender.send_timeout.to_i, 0].pack('L!L!')  # struct timeval
-        sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
-
-        sock
-      end
-
       def establish_connection(sock)
         while available? && @state != :established
           begin
@@ -430,98 +494,60 @@ module Fluent::Plugin
         end
       end
 
-      def send_data(tag, chunk)
-        sock = connect
+      def send_data_actual(sock, tag, chunk)
         @state = @sender.security ? :helo : :established
-        begin
-          set_socket_options(sock)
-
-          if @state != :established
-            establish_connection(sock)
-          end
+        if @state != :established
+          establish_connection(sock)
+        end
 
-          unless available?
-            raise ConnectionClosedError, "failed to establish connection with node #{@name}"
-          end
+        unless available?
+          raise ConnectionClosedError, "failed to establish connection with node #{@name}"
+        end
 
-          option = { 'size' => chunk.size, 'compressed' => @compress }
-          option['chunk'] = Base64.encode64(chunk.unique_id) if @sender.require_ack_response
+        option = { 'size' => chunk.size, 'compressed' => @compress }
+        option['chunk'] = Base64.encode64(chunk.unique_id) if @sender.require_ack_response
 
-          # out_forward always uses Raw32 type for content.
-          # Raw16 can store only 64kbytes, and it should be much smaller than buffer chunk size.
+        # out_forward always uses Raw32 type for content.
+        # Raw16 can store only 64kbytes, and it should be much smaller than buffer chunk size.
 
-          sock.write @sender.forward_header        # beginArray(3)
-          sock.write tag.to_msgpack                # 1. writeRaw(tag)
-          chunk.open(compressed: @compress) do |chunk_io|
-            sock.write [0xdb, chunk_io.size].pack('CN') # 2. beginRaw(size) raw32
-            IO.copy_stream(chunk_io, sock)              # writeRawBody(packed_es)
-          end
-          sock.write option.to_msgpack             # 3. writeOption(option)
-
-          if @sender.require_ack_response
-            # Waiting for a response here results in a decrease of throughput because a chunk queue is locked.
-            # To avoid a decrease of throughput, it is necessary to prepare a list of chunks that wait for responses
-            # and process them asynchronously.
-            if IO.select([sock], nil, nil, @sender.ack_response_timeout)
-              raw_data = begin
-                           sock.recv(1024)
-                         rescue Errno::ECONNRESET
-                           ""
-                         end
-
-              # When connection is closed by remote host, socket is ready to read and #recv returns an empty string that means EOF.
-              # If this happens we assume the data wasn't delivered and retry it.
-              if raw_data.empty?
-                @log.warn "node closed the connection. regard it as unavailable.", host: @host, port: @port
-                disable!
-                raise ConnectionClosedError, "node #{@host}:#{@port} closed connection"
-              else
-                @unpacker.feed(raw_data)
-                res = @unpacker.read
-                if res['ack'] != option['chunk']
-                  # Some errors may have occured when ack and chunk id is different, so send the chunk again.
-                  raise ResponseError, "ack in response and chunk id in sent data are different"
-                end
-              end
+        sock.write @sender.forward_header        # beginArray(3)
+        sock.write tag.to_msgpack                # 1. writeRaw(tag)
+        chunk.open(compressed: @compress) do |chunk_io|
+          sock.write [0xdb, chunk_io.size].pack('CN') # 2. beginRaw(size) raw32
+          IO.copy_stream(chunk_io, sock)              # writeRawBody(packed_es)
+        end
+        sock.write option.to_msgpack             # 3. writeOption(option)
+      end
 
-            else
-              # IO.select returns nil on timeout.
-              # There are 2 types of cases when no response has been received:
-              # (1) the node does not support sending responses
-              # (2) the node does support sending response but responses have not arrived for some reasons.
-              @log.warn "no response from node. regard it as unavailable.", host: @host, port: @port
-              disable!
-              raise ACKTimeoutError, "node #{host}:#{port} does not return ACK"
-            end
-          end
+      def send_data(tag, chunk)
+        sock = @sender.create_transfer_socket(resolved_host, port)
+        begin
+          send_data_actual(sock, tag, chunk)
+        rescue
+          sock.close rescue nil
+          raise
+        end
 
-          heartbeat(false)
-          res  # for test
-        ensure
-          sock.close_write
-          sock.close
+        if @sender.require_ack_response
+          return sock # to read ACK from socket
         end
+
+        sock.close_write rescue nil
+        sock.close rescue nil
+        heartbeat(false)
+        nil
       end
 
       # FORWARD_TCP_HEARTBEAT_DATA = FORWARD_HEADER + ''.to_msgpack + [].to_msgpack
       def send_heartbeat
         case @sender.heartbeat_type
         when :tcp
-          sock = connect
-          begin
-            opt = [1, @sender.send_timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
-            sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
-            # opt = [@sender.send_timeout.to_i, 0].pack('L!L!')  # struct timeval
-            # sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
-
+          @sender.create_transfer_socket(resolved_host, port) do |sock|
             ## don't send any data to not cause a compatibility problem
             # sock.write FORWARD_TCP_HEARTBEAT_DATA
 
             # successful tcp connection establishment is considered as valid heartbeat
             heartbeat(true)
-          ensure
-            sock.close_write
-            sock.close
           end
         when :udp
           @usock.send "\0", 0, Socket.pack_sockaddr_in(@port, resolved_host)
@@ -543,7 +569,7 @@ module Fluent::Plugin
           @resolved_host ||= resolve_dns!
 
         else
-          now = Engine.now
+          now = Fluent::Engine.now
           rh = @resolved_host
           if !rh || now - @resolved_time >= @sender.expire_dns_cache
             rh = @resolved_host = resolve_dns!