fluentd 0.14.7-x86-mingw32 → 0.14.10-x86-mingw32

data/lib/fluent/plugin/out_file.rb

@@ -16,42 +16,59 @@
 
 require 'fileutils'
 require 'zlib'
+require 'time'
 
-require 'fluent/output'
+require 'fluent/plugin/output'
 require 'fluent/config/error'
-require 'fluent/system_config'
+# TODO remove ...
+require 'fluent/plugin/file_util'
 
-module Fluent
-  class FileOutput < TimeSlicedOutput
-    include SystemConfig::Mixin
+module Fluent::Plugin
+  class FileOutput < Output
+    Fluent::Plugin.register_output('file', self)
 
-    Plugin.register_output('file', self)
+    helpers :formatter, :inject, :compat_parameters
 
-    SUPPORTED_COMPRESS = {
-      'gz' => :gz,
-      'gzip' => :gz,
+    SUPPORTED_COMPRESS = [:text, :gz, :gzip]
+    SUPPORTED_COMPRESS_MAP = {
+      text: nil,
+      gz: :gzip,
+      gzip: :gzip,
     }
 
     FILE_PERMISSION = 0644
     DIR_PERMISSION = 0755
 
+    DEFAULT_TIMEKEY = 60 * 60 * 24
+
     desc "The Path of the file."
     config_param :path, :string
-    desc "The format of the file content. The default is out_file."
-    config_param :format, :string, default: 'out_file', skip_accessor: true
+
+    desc "Specify to add file suffix for bare file path or not."
+    config_param :add_path_suffix, :bool, default: true
+    desc "The file suffix added to bare file path."
+    config_param :path_suffix, :string, default: '.log'
     desc "The flushed chunk is appended to existence file or not."
     config_param :append, :bool, default: false
     desc "Compress flushed file."
-    config_param :compress, default: nil do |val|
-      c = SUPPORTED_COMPRESS[val]
-      unless c
-        raise ConfigError, "Unsupported compression algorithm '#{val}'"
-      end
-      c
-    end
-    desc "Create symlink to temporary buffered file when buffer_type is file."
+    config_param :compress, :enum, list: SUPPORTED_COMPRESS, default: :text
+    desc "Execute compression again even when buffer chunk is already compressed."
+    config_param :recompress, :bool, default: false
+    desc "Create symlink to temporary buffered file when buffer_type is file (disabled on Windows)."
     config_param :symlink_path, :string, default: nil
 
+    config_section :format do
+      config_set_default :@type, 'out_file'
+    end
+
+    config_section :buffer do
+      config_set_default :@type, 'file'
+      config_set_default :chunk_keys, ['time']
+      config_set_default :timekey, DEFAULT_TIMEKEY
+    end
+
+    attr_accessor :last_written_path # for tests
+
     module SymlinkBufferMixin
       def symlink_path=(path)
         @_symlink_path = path
@@ -63,52 +80,81 @@ module Fluent
         # timekey will be appended into that file chunk. On the other side, resumed file chunks might NOT
         # have timekey, especially in the cases that resumed file chunks are generated by Fluentd v0.12.
         # These chunks will be enqueued immediately, and will be flushed soon.
-        latest_chunk = metadata_list.select{|m| m.timekey }.sort_by(&:timekey).last
-        if chunk.metadata == latest_chunk
+        latest_metadata = metadata_list.select{|m| m.timekey }.sort_by(&:timekey).last
+        if chunk.metadata == latest_metadata
           FileUtils.ln_sf(chunk.path, @_symlink_path)
         end
         chunk
       end
     end
 
-    def initialize
-      require 'zlib'
-      require 'time'
-      require 'fluent/plugin/file_util'
-      super
-    end
-
     def configure(conf)
-      if path = conf['path']
-        @path = path
+      compat_parameters_convert(conf, :formatter, :buffer, :inject, default_chunk_key: "time")
+
+      configured_time_slice_format = conf['time_slice_format']
+
+      # v0.14 file buffer handles path as directory if '*' is missing
+      # 'dummy_path' is not to raise configuration error for 'path' in file buffer plugin,
+      # but raise it in this plugin.
+      if conf.elements(name: 'buffer').empty?
+        conf.add_element('buffer', 'time')
       end
-      unless @path
-        raise ConfigError, "'path' parameter is required on file output"
+      buffer_conf = conf.elements(name: 'buffer').first
+      unless buffer_conf.has_key?('path')
+        buffer_conf['path'] = conf['path'] || '/tmp/dummy_path'
       end
 
-      if pos = @path.index('*')
-        @path_prefix = @path[0,pos]
-        @path_suffix = @path[pos+1..-1]
-        conf['buffer_path'] ||= "#{@path}"
-      else
-        @path_prefix = @path+"."
-        @path_suffix = ".log"
-        conf['buffer_path'] ||= "#{@path}.*"
-      end
+      super
+
+      @compress_method = SUPPORTED_COMPRESS_MAP[@compress]
 
-      test_path = generate_path(Time.now.strftime(@time_slice_format))
-      unless ::Fluent::FileUtil.writable_p?(test_path)
-        raise ConfigError, "out_file: `#{test_path}` is not writable"
+      if @path.include?('*') && !@buffer_config.timekey
+        raise Fluent::ConfigError, "path including '*' must be used with buffer chunk key 'time'"
       end
 
-      super
+      path_suffix = @add_path_suffix ? @path_suffix : ''
+      path_timekey = if @chunk_key_time
+                       @as_secondary ? @primary_instance.buffer_config.timekey : @buffer_config.timekey
+                     else
+                       nil
+                     end
+      @path_template = generate_path_template(@path, path_timekey, @append, @compress_method, path_suffix: path_suffix, time_slice_format: configured_time_slice_format)
 
-      @formatter = Plugin.new_formatter(@format)
-      @formatter.configure(conf)
+      if @as_secondary
+        # When this plugin is configured as secondary & primary plugin has tag key, but this plugin may not have it.
+        # Increment placeholder can make another output file per chunk tag/keys even if original path doesn't include it.
+        placeholder_validators(:path, @path_template).select{|v| v.type == :time }.each do |v|
+          v.validate!
+        end
+      else
+        placeholder_validate!(:path, @path_template)
+
+        max_tag_index = get_placeholders_tag(@path_template).max || 1
+        max_tag_index = 1 if max_tag_index < 1
+        dummy_tag = (['a'] * max_tag_index).join('.')
+        dummy_record_keys = get_placeholders_keys(@path_template) || ['message']
+        dummy_record = Hash[dummy_record_keys.zip(['data'] * dummy_record_keys.size)]
+
+        test_meta1 = metadata_for_test(dummy_tag, Fluent::Engine.now, dummy_record)
+        test_path = extract_placeholders(@path_template, test_meta1)
+        unless ::Fluent::FileUtil.writable_p?(test_path)
+          raise Fluent::ConfigError, "out_file: `#{test_path}` is not writable"
+        end
+      end
+
+      @formatter = formatter_create
 
       if @symlink_path && @buffer.respond_to?(:path)
-        @buffer.extend SymlinkBufferMixin
-        @buffer.symlink_path = @symlink_path
+        if @as_secondary
+          raise Fluent::ConfigError, "symlink_path option is unavailable in <secondary>: consider to use secondary_file plugin"
+        end
+        if Fluent.windows?
+          log.warn "symlink_path is unavailable on Windows platform. disabled."
+          @symlink_path = nil
+        else
+          @buffer.extend SymlinkBufferMixin
+          @buffer.symlink_path = @symlink_path
+        end
       end
 
       @dir_perm = system_config.dir_permission || DIR_PERMISSION
@@ -116,56 +162,101 @@ module Fluent
     end
 
     def format(tag, time, record)
-      @formatter.format(tag, time, record)
+      r = inject_values_to_record(tag, time, record)
+      @formatter.format(tag, time, r)
     end
 
     def write(chunk)
-      path = generate_path(chunk.key)
+      path = extract_placeholders(@path_template, chunk.metadata)
       FileUtils.mkdir_p File.dirname(path), mode: @dir_perm
 
-      case @compress
+      unless @append
+        path = find_filepath_available(path)
+      end
+
+      case @compress_method
       when nil
-        File.open(path, "ab", @file_perm) {|f|
+        File.open(path, "ab", @file_perm) do |f|
           chunk.write_to(f)
-        }
-      when :gz
-        File.open(path, "ab", @file_perm) {|f|
-          gz = Zlib::GzipWriter.new(f)
-          chunk.write_to(gz)
-          gz.close
-        }
+        end
+      when :gzip
+        if @buffer.compress != :gzip || @recompress
+          File.open(path, "ab", @file_perm) do |f|
+            gz = Zlib::GzipWriter.new(f)
+            chunk.write_to(gz, compressed: :text)
+            gz.close
+          end
+        else
+          File.open(path, "ab", @file_perm) do |f|
+            chunk.write_to(f, compressed: :gzip)
+          end
+        end
+      else
+        raise "BUG: unknown compression method #{@compress_method}"
       end
 
-      return path  # for test
+      @last_written_path = path
     end
 
-    def secondary_init(primary)
-      # don't warn even if primary.class is not FileOutput
+    def timekey_to_timeformat(timekey)
+      case timekey
+      when nil          then ''
+      when 0...60       then '%Y%m%d%H%M%S' # 60 exclusive
+      when 60...3600    then '%Y%m%d%H%M'
+      when 3600...86400 then '%Y%m%d%H'
+      else                   '%Y%m%d'
+      end
     end
 
-    private
-
-    def suffix
-      case @compress
-      when nil
-        ''
-      when :gz
-        ".gz"
+    def compression_suffix(compress)
+      case compress
+      when :gzip then '.gz'
+      when nil then ''
+      else
+        raise ArgumentError, "unknown compression type #{compress}"
       end
     end
 
-    def generate_path(time_string)
-      if @append
-        "#{@path_prefix}#{time_string}#{@path_suffix}#{suffix}"
+    # /path/to/dir/file.*      -> /path/to/dir/file.%Y%m%d
+    # /path/to/dir/file.*.data -> /path/to/dir/file.%Y%m%d.data
+    # /path/to/dir/file        -> /path/to/dir/file.%Y%m%d.log
+    #   %Y%m%d -> %Y%m%d_** (non append)
+    # + .gz (gzipped)
+    ## TODO: remove time_slice_format when end of support of compat_parameters
+    def generate_path_template(original, timekey, append, compress, path_suffix: '', time_slice_format: nil)
+      comp_suffix = compression_suffix(compress)
+      index_placeholder = append ? '' : '_**'
+      if original.index('*')
+        raise "BUG: configuration error must be raised for path including '*' without timekey" unless timekey
+        time_placeholders_part = time_slice_format || timekey_to_timeformat(timekey)
+        original.gsub('*', time_placeholders_part + index_placeholder) + comp_suffix
       else
-        path = nil
-        i = 0
-        begin
-          path = "#{@path_prefix}#{time_string}_#{i}#{@path_suffix}#{suffix}"
-          i += 1
-        end while File.exist?(path)
-        path
+        if timekey
+          if time_slice_format
+            "#{original}.#{time_slice_format}#{index_placeholder}#{path_suffix}#{comp_suffix}"
+          else
+            time_placeholders = timekey_to_timeformat(timekey)
+            if time_placeholders.scan(/../).any?{|ph| original.include?(ph) }
+              raise Fluent::ConfigError, "insufficient timestamp placeholders in path" if time_placeholders.scan(/../).any?{|ph| !original.include?(ph) }
+              "#{original}#{index_placeholder}#{path_suffix}#{comp_suffix}"
+            else
+              "#{original}.#{time_placeholders}#{index_placeholder}#{path_suffix}#{comp_suffix}"
+            end
+          end
+        else
+          "#{original}#{index_placeholder}#{path_suffix}#{comp_suffix}"
+        end
+      end
+    end
+
+    def find_filepath_available(path_with_placeholder) # for non-append
+      raise "BUG: index placeholder not found in path: #{path_with_placeholder}" unless path_with_placeholder.index('_**')
+      i = 0
+      while path = path_with_placeholder.sub('_**', "_#{i}")
+        break unless File.exist?(path)
+        i += 1
       end
+      path
     end
   end
 end

data/lib/fluent/plugin/out_forward.rb

@@ -14,44 +14,30 @@
 #    limitations under the License.
 #
 
-require 'base64'
-require 'socket'
-require 'fileutils'
-
-require 'cool.io'
-
 require 'fluent/output'
 require 'fluent/config/error'
+require 'base64'
 
-module Fluent
-  class ForwardOutputError < StandardError
-  end
-
-  class ForwardOutputResponseError < ForwardOutputError
-  end
+require 'fluent/compat/socket_util'
 
-  class ForwardOutputConnectionClosedError < ForwardOutputError
-  end
+module Fluent::Plugin
+  class ForwardOutput < Output
+    class Error < StandardError; end
+    class NoNodesAvailable < Error; end
+    class ConnectionClosedError < Error; end
 
-  class ForwardOutputACKTimeoutError < ForwardOutputResponseError
-  end
+    Fluent::Plugin.register_output('forward', self)
 
-  class ForwardOutput < ObjectBufferedOutput
-    Plugin.register_output('forward', self)
+    helpers :socket, :server, :timer, :thread, :compat_parameters
 
     LISTEN_PORT = 24224
 
-    def initialize
-      super
-      require 'fluent/plugin/socket_util'
-      @nodes = []  #=> [Node]
-      @loop = nil
-      @thread = nil
-      @finished = false
-    end
+    PROCESS_CLOCK_ID = Process::CLOCK_MONOTONIC_RAW rescue Process::CLOCK_MONOTONIC
 
     desc 'The timeout time when sending event logs.'
     config_param :send_timeout, :time, default: 60
+    # TODO: add linger_timeout, recv_timeout
+
     desc 'The transport protocol to use for heartbeats.(udp,tcp,none)'
     config_param :heartbeat_type, :enum, list: [:tcp, :udp, :none], default: :tcp
     desc 'The interval of the heartbeat packer.'
@@ -60,8 +46,6 @@ module Fluent
     config_param :recover_wait, :time, default: 10
     desc 'The hard timeout used to detect server failure.'
     config_param :hard_timeout, :time, default: 60
-    desc 'Set TTL to expire DNS cache in seconds.'
-    config_param :expire_dns_cache, :time, default: nil  # 0 means disable cache
     desc 'The threshold parameter used to detect server faults.'
     config_param :phi_threshold, :integer, default: 16
     desc 'Use the "Phi accrual failure detector" to detect server failure.'
@@ -69,14 +53,20 @@ module Fluent
 
     desc 'Change the protocol to at-least-once.'
     config_param :require_ack_response, :bool, default: false  # require in_forward to respond with ack
+
+    ## The reason of default value of :ack_response_timeout:
+    # Linux default tcp_syn_retries is 5 (in many environment)
+    # 3 + 6 + 12 + 24 + 48 + 96 -> 189 (sec)
     desc 'This option is used when require_ack_response is true.'
     config_param :ack_response_timeout, :time, default: 190
-    desc 'Reading data size from server'
-    config_param :read_length, :size, default: 512 # 512bytes
+
     desc 'The interval while reading data from server'
     config_param :read_interval_msec, :integer, default: 50 # 50ms
-    # Linux default tcp_syn_retries is 5 (in many environment)
-    # 3 + 6 + 12 + 24 + 48 + 96 -> 189 (sec)
+    desc 'Reading data size from server'
+    config_param :read_length, :size, default: 512 # 512bytes
+
+    desc 'Set TTL to expire DNS cache in seconds.'
+    config_param :expire_dns_cache, :time, default: nil  # 0 means disable cache
     desc 'Enable client-side DNS round robin.'
     config_param :dns_round_robin, :bool, default: false # heartbeat_type 'udp' is not available for this
 
@@ -114,17 +104,39 @@ module Fluent
     config_param :port, :integer, default: LISTEN_PORT, obsoleted: "User <server> section instead."
     config_param :host, :string, default: nil, obsoleted: "Use <server> section instead."
 
+    config_section :buffer do
+      config_set_default :chunk_keys, ["tag"]
+    end
+
     attr_reader :read_interval, :recover_sample_size
 
+    def initialize
+      super
+
+      @nodes = [] #=> [Node]
+      @loop = nil
+      @thread = nil
+
+      @usock = nil
+      @sock_ack_waiting = nil
+      @sock_ack_waiting_mutex = nil
+    end
+
     def configure(conf)
+      compat_parameters_convert(conf, :buffer, default_chunk_key: 'tag')
+
       super
 
+      unless @chunk_key_tag
+        raise Fluent::ConfigError, "buffer chunk key must include 'tag' for forward output"
+      end
+
       @read_interval = @read_interval_msec / 1000.0
       @recover_sample_size = @recover_wait / @heartbeat_interval
 
       if @dns_round_robin
         if @heartbeat_type == :udp
-          raise ConfigError, "forward output heartbeat type must be 'tcp' or 'none' to use dns_round_robin option"
+          raise Fluent::ConfigError, "forward output heartbeat type must be 'tcp' or 'none' to use dns_round_robin option"
         end
       end
 
@@ -140,90 +152,119 @@ module Fluent
         end
       end
 
-      if @compress == :gzip && @buffer.compress == :text
-        @buffer.compress = :gzip
-      elsif @compress == :text && @buffer.compress == :gzip
-        log.info "buffer is compressed.  If you also want to save the bandwidth of a network, Add `compress` configuration in <match>"
+      unless @as_secondary
+        if @compress == :gzip && @buffer.compress == :text
+          @buffer.compress = :gzip
+        elsif @compress == :text && @buffer.compress == :gzip
+          log.info "buffer is compressed.  If you also want to save the bandwidth of a network, Add `compress` configuration in <match>"
+        end
       end
 
       if @nodes.empty?
-        raise ConfigError, "forward output plugin requires at least one <server> is required"
+        raise Fluent::ConfigError, "forward output plugin requires at least one <server> is required"
       end
 
       raise Fluent::ConfigError, "ack_response_timeout must be a positive integer" if @ack_response_timeout < 1
     end
 
+    def prefer_delayed_commit
+      @require_ack_response
+    end
+
     def start
       super
 
+      # Output#start sets @delayed_commit_timeout by @buffer_config.delayed_commit_timeout
+      # But it should be overwritten by ack_response_timeout to rollback chunks after timeout
+      if @ack_response_timeout && @delayed_commit_timeout != @ack_response_timeout
+        log.info "delayed_commit_timeout is overwritten by ack_response_timeout"
+        @delayed_commit_timeout = @ack_response_timeout
+      end
+
       @rand_seed = Random.new.seed
       rebuild_weight_array
       @rr = 0
-      @usock = nil
 
       unless @heartbeat_type == :none
-        @loop = Coolio::Loop.new
-
         if @heartbeat_type == :udp
-          # assuming all hosts use udp
-          @usock = SocketUtil.create_udp_socket(@nodes.first.host)
-          @usock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK)
-          @hb = HeartbeatHandler.new(@usock, method(:on_heartbeat))
-          @loop.attach(@hb)
+          @usock = socket_create_udp(@nodes.first.host, @nodes.first.port, nonblock: true)
+          server_create_udp(:out_forward_heartbeat_receiver, 0, socket: @usock, max_bytes: @read_length) do |data, sock|
+            sockaddr = Socket.pack_sockaddr_in(sock.remote_port, sock.remote_host)
+            on_heartbeat(sockaddr, data)
+          end
         end
+        timer_execute(:out_forward_heartbeat_request, @heartbeat_interval, &method(:on_timer))
+      end
 
-        @timer = HeartbeatRequestTimer.new(@heartbeat_interval, method(:on_timer))
-        @loop.attach(@timer)
-
-        @thread = Thread.new(&method(:run))
+      if @require_ack_response
+        @sock_ack_waiting_mutex = Mutex.new
+        @sock_ack_waiting = []
+        thread_create(:out_forward_receiving_ack, &method(:ack_reader))
       end
     end
 
-    def shutdown
-      @finished = true
-      if @loop
-        @loop.watchers.each {|w| w.detach }
-        @loop.stop
-      end
-      @thread.join if @thread
+    def close
       @usock.close if @usock
-
       super
     end
 
-    def run
-      @loop.run if @loop
-    rescue
-      log.error "unexpected error", error: $!.to_s
-      log.error_backtrace
+    def write(chunk)
+      return if chunk.empty?
+      tag = chunk.metadata.tag
+      select_a_healthy_node{|node| node.send_data(tag, chunk) }
     end
 
-    def write_objects(tag, chunk)
-      return if chunk.empty?
+    ACKWaitingSockInfo = Struct.new(:sock, :chunk_id, :node, :time, :timeout) do
+      def expired?(now)
+        time + timeout < now
+      end
+    end
+
+    def try_write(chunk)
+      if chunk.empty?
+        commit_write(chunk.unique_id)
+        return
+      end
+      tag = chunk.metadata.tag
+      sock, node = select_a_healthy_node{|n| n.send_data(tag, chunk) }
+      chunk_id = Base64.encode64(chunk.unique_id)
+      current_time = Process.clock_gettime(PROCESS_CLOCK_ID)
+      info = ACKWaitingSockInfo.new(sock, chunk_id, node, current_time, @ack_response_timeout)
+      @sock_ack_waiting_mutex.synchronize do
+        @sock_ack_waiting << info
+      end
+    end
 
+    def select_a_healthy_node
       error = nil
 
       wlen = @weight_array.length
       wlen.times do
         @rr = (@rr + 1) % wlen
         node = @weight_array[@rr]
+        next unless node.available?
 
-        if node.available?
-          begin
-            node.send_data(tag, chunk)
-            return
-          rescue
-            # for load balancing during detecting crashed servers
-            error = $!  # use the latest error
-          end
+        begin
+          ret = yield node
+          return ret, node
+        rescue
+          # for load balancing during detecting crashed servers
+          error = $!  # use the latest error
         end
       end
 
-      if error
-        raise error
-      else
-        raise "no nodes are available"  # TODO message
-      end
+      raise error if error
+      raise NoNodesAvailable, "no nodes are available"
+    end
+
+    def create_transfer_socket(host, port, &block)
+      socket_create_tcp(
+        host, port,
+        linger_timeout: @send_timeout,
+        send_timeout: @send_timeout,
+        recv_timeout: @ack_response_timeout,
+        &block
+      )
     end
 
     # MessagePack FixArray length is 3
@@ -276,21 +317,7 @@ module Fluent
       @weight_array = weight_array
     end
 
-    class HeartbeatRequestTimer < Coolio::TimerWatcher
-      def initialize(interval, callback)
-        super(interval, true)
-        @callback = callback
-      end
-
-      def on_timer
-        @callback.call
-      rescue
-        # TODO log?
-      end
-    end
-
     def on_timer
-      return if @finished
       @nodes.each {|n|
         if n.tick
           rebuild_weight_array
@@ -305,33 +332,86 @@ module Fluent
       }
     end
 
-    class HeartbeatHandler < Coolio::IO
-      def initialize(io, callback)
-        super(io)
-        @io = io
-        @callback = callback
+    def on_heartbeat(sockaddr, msg)
+      if node = @nodes.find {|n| n.sockaddr == sockaddr }
+        # log.trace "heartbeat arrived", name: node.name, host: node.host, port: node.port
+        if node.heartbeat
+          rebuild_weight_array
+        end
       end
+    end
 
-      def on_readable
-        begin
-          msg, addr = @io.recvfrom(1024)
-        rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR
-          return
+    # return chunk id when succeeded for tests
+    def read_ack_from_sock(sock, unpacker)
+      begin
+        raw_data = sock.recv(@read_length)
+      rescue Errno::ECONNRESET
+        raw_data = ""
+      end
+      info = @sock_ack_waiting_mutex.synchronize{ @sock_ack_waiting.find{|i| i.sock == sock } }
+
+      # When connection is closed by remote host, socket is ready to read and #recv returns an empty string that means EOF.
+      # If this happens we assume the data wasn't delivered and retry it.
+      if raw_data.empty?
+        log.warn "destination node closed the connection. regard it as unavailable.", host: info.node.host, port: info.node.port
+        info.node.disable!
+        return nil
+      else
+        unpacker.feed(raw_data)
+        res = unpacker.read
+        if res['ack'] != info.chunk_id
+          # Some errors may have occured when ack and chunk id is different, so send the chunk again.
+          log.warn "ack in response and chunk id in sent data are different", chunk_id: info.chunk_id, ack: res['ack']
+          rollback_write(info.chunk_id)
+          return nil
         end
-        host = addr[3]
-        port = addr[1]
-        sockaddr = Socket.pack_sockaddr_in(port, host)
-        @callback.call(sockaddr, msg)
-      rescue
-        # TODO log?
+        return info.chunk_id
+      end
+    rescue => e
+      log.error "unexpected error while receiving ack message", error: e
+      log.error_backtrace
+    ensure
+      @sock_ack_waiting_mutex.synchronize do
+        @sock_ack_waiting.delete(info)
       end
     end
 
-    def on_heartbeat(sockaddr, msg)
-      if node = @nodes.find {|n| n.sockaddr == sockaddr }
-        #log.trace "heartbeat from '#{node.name}'", :host=>node.host, :port=>node.port
-        if node.heartbeat
-          rebuild_weight_array
+    def ack_reader
+      select_interval = if @delayed_commit_timeout > 3
+                          2
+                        else
+                          @delayed_commit_timeout / 2.0
+                        end
+
+      unpacker = Fluent::Engine.msgpack_unpacker
+
+      while thread_current_running?
+        now = Process.clock_gettime(PROCESS_CLOCK_ID)
+        sockets = []
+        @sock_ack_waiting_mutex.synchronize do
+          new_list = []
+          @sock_ack_waiting.each do |info|
+            if info.expired?(now)
+              # There are 2 types of cases when no response has been received from socket:
+              # (1) the node does not support sending responses
+              # (2) the node does support sending response but responses have not arrived for some reasons.
+              log.warn "no response from node. regard it as unavailable.", host: info.node.host, port: info.node.port
+              info.node.disable!
+              info.sock.close rescue nil
+              rollback_write(info.chunk_id)
+            else
+              sockets << info.sock
+              new_list << info
+            end
+          end
+          @sock_ack_waiting = new_list
+        end
+
+        readable_sockets, _, _ = IO.select(sockets, nil, nil, select_interval)
+        next unless readable_sockets
+
+        readable_sockets.each do |sock|
+          read_ack_from_sock(sock, unpacker)
         end
       end
     end
@@ -384,20 +464,6 @@ module Fluent
         @standby
       end
 
-      def connect
-        TCPSocket.new(resolved_host, port)
-      end
-
-      def set_socket_options(sock)
-        opt = [1, @sender.send_timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
-        sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
-
-        opt = [@sender.send_timeout.to_i, 0].pack('L!L!')  # struct timeval
-        sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
-
-        sock
-      end
-
       def establish_connection(sock)
         while available? && @state != :established
           begin
@@ -428,98 +494,60 @@ module Fluent
         end
       end
 
-      def send_data(tag, chunk)
-        sock = connect
+      def send_data_actual(sock, tag, chunk)
         @state = @sender.security ? :helo : :established
-        begin
-          set_socket_options(sock)
-
-          if @state != :established
-            establish_connection(sock)
-          end
+        if @state != :established
+          establish_connection(sock)
+        end
 
-          unless available?
-            raise ForwardOutputConnectionClosedError, "failed to establish connection with node #{@name}"
-          end
+        unless available?
+          raise ConnectionClosedError, "failed to establish connection with node #{@name}"
+        end
 
-          option = { 'size' => chunk.size_of_events, 'compressed' => @compress }
-          option['chunk'] = Base64.encode64(chunk.unique_id) if @sender.require_ack_response
+        option = { 'size' => chunk.size, 'compressed' => @compress }
+        option['chunk'] = Base64.encode64(chunk.unique_id) if @sender.require_ack_response
 
-          # out_forward always uses Raw32 type for content.
-          # Raw16 can store only 64kbytes, and it should be much smaller than buffer chunk size.
+        # out_forward always uses Raw32 type for content.
+        # Raw16 can store only 64kbytes, and it should be much smaller than buffer chunk size.
 
-          sock.write @sender.forward_header        # beginArray(3)
-          sock.write tag.to_msgpack                # 1. writeRaw(tag)
-          chunk.open(compressed: @compress) do |chunk_io|
-            sock.write [0xdb, chunk_io.size].pack('CN') # 2. beginRaw(size) raw32
-            IO.copy_stream(chunk_io, sock)              # writeRawBody(packed_es)
-          end
-          sock.write option.to_msgpack             # 3. writeOption(option)
-
-          if @sender.require_ack_response
-            # Waiting for a response here results in a decrease of throughput because a chunk queue is locked.
-            # To avoid a decrease of throughput, it is necessary to prepare a list of chunks that wait for responses
-            # and process them asynchronously.
-            if IO.select([sock], nil, nil, @sender.ack_response_timeout)
-              raw_data = begin
-                           sock.recv(1024)
-                         rescue Errno::ECONNRESET
-                           ""
-                         end
-
-              # When connection is closed by remote host, socket is ready to read and #recv returns an empty string that means EOF.
-              # If this happens we assume the data wasn't delivered and retry it.
-              if raw_data.empty?
-                @log.warn "node closed the connection. regard it as unavailable.", host: @host, port: @port
-                disable!
-                raise ForwardOutputConnectionClosedError, "node #{@host}:#{@port} closed connection"
-              else
-                @unpacker.feed(raw_data)
-                res = @unpacker.read
-                if res['ack'] != option['chunk']
-                  # Some errors may have occured when ack and chunk id is different, so send the chunk again.
-                  raise ForwardOutputResponseError, "ack in response and chunk id in sent data are different"
-                end
-              end
+        sock.write @sender.forward_header        # beginArray(3)
+        sock.write tag.to_msgpack                # 1. writeRaw(tag)
+        chunk.open(compressed: @compress) do |chunk_io|
+          sock.write [0xdb, chunk_io.size].pack('CN') # 2. beginRaw(size) raw32
+          IO.copy_stream(chunk_io, sock)              # writeRawBody(packed_es)
+        end
+        sock.write option.to_msgpack             # 3. writeOption(option)
+      end
 
-            else
-              # IO.select returns nil on timeout.
-              # There are 2 types of cases when no response has been received:
-              # (1) the node does not support sending responses
-              # (2) the node does support sending response but responses have not arrived for some reasons.
-              @log.warn "no response from node. regard it as unavailable.", host: @host, port: @port
-              disable!
-              raise ForwardOutputACKTimeoutError, "node #{host}:#{port} does not return ACK"
-            end
-          end
+      def send_data(tag, chunk)
+        sock = @sender.create_transfer_socket(resolved_host, port)
+        begin
+          send_data_actual(sock, tag, chunk)
+        rescue
+          sock.close rescue nil
+          raise
+        end
 
-          heartbeat(false)
-          res  # for test
-        ensure
-          sock.close_write
-          sock.close
+        if @sender.require_ack_response
+          return sock # to read ACK from socket
         end
+
+        sock.close_write rescue nil
+        sock.close rescue nil
+        heartbeat(false)
+        nil
       end
 
       # FORWARD_TCP_HEARTBEAT_DATA = FORWARD_HEADER + ''.to_msgpack + [].to_msgpack
       def send_heartbeat
         case @sender.heartbeat_type
         when :tcp
-          sock = connect
-          begin
-            opt = [1, @sender.send_timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
-            sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
-            # opt = [@sender.send_timeout.to_i, 0].pack('L!L!')  # struct timeval
-            # sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
-
+          @sender.create_transfer_socket(resolved_host, port) do |sock|
             ## don't send any data to not cause a compatibility problem
             # sock.write FORWARD_TCP_HEARTBEAT_DATA
 
             # successful tcp connection establishment is considered as valid heartbeat
             heartbeat(true)
-          ensure
-            sock.close_write
-            sock.close
           end
         when :udp
           @usock.send "\0", 0, Socket.pack_sockaddr_in(@port, resolved_host)
@@ -541,7 +569,7 @@ module Fluent
           @resolved_host ||= resolve_dns!
 
         else
-          now = Engine.now
+          now = Fluent::Engine.now
           rh = @resolved_host
           if !rh || now - @resolved_time >= @sender.expire_dns_cache
             rh = @resolved_host = resolve_dns!
@@ -601,11 +629,6 @@ module Fluent
         end
       end
 
-      # TODO: #to_msgpack(string) is deprecated
-      def to_msgpack(out = '')
-        [@host, @port, @weight, @available].to_msgpack(out)
-      end
-
       def generate_salt
         SecureRandom.hex(16)
       end