fluentd 0.14.4 → 0.14.5

data/lib/fluent/plugin/out_null.rb

@@ -18,10 +18,47 @@ require 'fluent/plugin/output'
 
 module Fluent::Plugin
   class NullOutput < Output
+    # This plugin is for tests of non-buffered/buffered plugins
     Fluent::Plugin.register_output('null', self)
 
+    config_section :buffer do
+      config_set_default :chunk_keys, ['tag']
+      config_set_default :flush_at_shutdown, true
+      config_set_default :chunk_limit_size, 10 * 1024
+    end
+
+    def prefer_buffered_processing
+      false
+    end
+
+    def prefer_delayed_commit
+      @delayed
+    end
+
+    attr_accessor :feed_proc, :delayed
+
+    def initialize
+      super
+      @delayed = false
+      @feed_proc = nil
+    end
+
     def process(tag, es)
       # Do nothing
     end
+
+    def write(chunk)
+      if @feed_proc
+        @feed_proc.call(chunk)
+      end
+    end
+
+    def try_write(chunk)
+      if @feed_proc
+        @feed_proc.call(chunk)
+      end
+      # not to commit chunks for testing
+      # commit_write(chunk.unique_id)
+    end
   end
 end

data/lib/fluent/plugin/out_secondary_file.rb

@@ -0,0 +1,128 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require "fileutils"
+require "fluent/plugin/file_util"
+require "fluent/plugin/output"
+require "fluent/config/error"
+
+module Fluent::Plugin
+  class SecondaryFileOutput < Output
+    Fluent::Plugin.register_output("secondary_file", self)
+
+    FILE_PERMISSION = 0644
+    DIR_PERMISSION = 0755
+    PLACEHOLDER_REGEX = /\${(tag(\[\d+\])?|[\w.@-]+)}/
+
+    desc "The directory path of the output file."
+    config_param :directory, :string
+    desc "The basename of the output file."
+    config_param :basename, :string, default: "dump.bin"
+    desc "The flushed chunk is appended to existence file or not."
+    config_param :append, :bool, default: false
+    config_param :compress, :enum, list: [:text, :gzip], default: :text
+
+    def configure(conf)
+      super
+
+      unless @as_secondary
+        raise Fluent::ConfigError, "This plugin can only be used in the <secondary> section"
+      end
+
+      if @basename.include?("/")
+        raise Fluent::ConfigError, "basename should not include `/`"
+      end
+
+      @path_without_suffix = File.join(@directory, @basename)
+      validate_compatible_with_primary_buffer!(@path_without_suffix)
+
+      @suffix = case @compress
+                when :text
+                  ""
+                when :gzip
+                  ".gz"
+                end
+
+      test_path = @path_without_suffix
+      unless Fluent::FileUtil.writable_p?(test_path)
+        raise Fluent::ConfigError, "out_secondary_file: `#{@directory}` should be writable"
+      end
+
+      @dir_perm = system_config.dir_permission || DIR_PERMISSION
+      @file_perm = system_config.file_permission || FILE_PERMISSION
+    end
+
+    def write(chunk)
+      path_without_suffix = extract_placeholders(@path_without_suffix, chunk.metadata)
+      path = generate_path(path_without_suffix)
+      FileUtils.mkdir_p File.dirname(path), mode: @dir_perm
+
+      case @compress
+      when :text
+        File.open(path, "ab", @file_perm) {|f|
+          f.flock(File::LOCK_EX)
+          chunk.write_to(f)
+        }
+      when :gzip
+        File.open(path, "ab", @file_perm) {|f|
+          f.flock(File::LOCK_EX)
+          gz = Zlib::GzipWriter.new(f)
+          chunk.write_to(gz)
+          gz.close
+        }
+      end
+
+      path
+    end
+
+    private
+
+    def validate_compatible_with_primary_buffer!(path_without_suffix)
+      placeholders = path_without_suffix.scan(PLACEHOLDER_REGEX).flat_map(&:first) # to trim suffix [\d+]
+
+      if !@chunk_key_time && has_time_format?(path_without_suffix)
+        raise Fluent::ConfigError, "out_secondary_file: basename or directory has an incompatible placeholder, remove time formats, like `%Y%m%d`, from basename or directory"
+      end
+
+      if !@chunk_key_tag && (ph = placeholders.find { |placeholder| placeholder.match(/tag(\[\d+\])?/) })
+        raise Fluent::ConfigError, "out_secondary_file: basename or directory has an incompatible placeholder #{ph}, remove tag placeholder, like `${tag}`, from basename or directory"
+      end
+
+      vars = placeholders.reject { |placeholder| placeholder.match(/tag(\[\d+\])?/) }
+
+      if ph = vars.find { |v| !@chunk_keys.include?(v) }
+        raise Fluent::ConfigError, "out_secondary_file: basename or directory has an incompatible placeholder #{ph}, remove variable placeholder, like `${varname}`, from basename or directory"
+      end
+    end
+
+    def has_time_format?(str)
+      str != Time.now.strftime(str)
+    end
+
+    def generate_path(path_without_suffix)
+      if @append
+        "#{path_without_suffix}#{@suffix}"
+      else
+        i = 0
+        loop do
+          path = "#{path_without_suffix}.#{i}#{@suffix}"
+          return path unless File.exist?(path)
+          i += 1
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_stdout.rb

@@ -24,25 +24,61 @@ module Fluent::Plugin
 
     DEFAULT_FORMAT_TYPE = 'json'
 
+    config_section :buffer do
+      config_set_default :chunk_keys, ['tag']
+      config_set_default :flush_at_shutdown, true
+      config_set_default :chunk_limit_size, 10 * 1024
+    end
+
     config_section :format do
       config_set_default :@type, DEFAULT_FORMAT_TYPE
     end
 
+    def prefer_buffered_processing
+      false
+    end
+
+    def prefer_delayed_commit
+      @delayed
+    end
+
+    attr_accessor :delayed
+
+    def initialize
+      super
+      @delayed = false
+    end
+
     def configure(conf)
       if conf['output_type'] && !conf['format']
         conf['format'] = conf['output_type']
       end
       compat_parameters_convert(conf, :inject, :formatter)
+
       super
+
       @formatter = formatter_create(conf: conf.elements('format').first, default_type: DEFAULT_FORMAT_TYPE)
     end
 
     def process(tag, es)
       es.each {|time,record|
-        r = inject_values_to_record(tag, time, record)
-        $log.write "#{Time.at(time).localtime} #{tag}: #{@formatter.format(tag, time, r).chomp}\n"
+        $log.write(format(tag, time, record))
       }
       $log.flush
     end
+
+    def format(tag, time, record)
+      record = inject_values_to_record(tag, time, record)
+      "#{Time.at(time).localtime} #{tag}: #{@formatter.format(tag, time, record).chomp}\n"
+    end
+
+    def write(chunk)
+      chunk.write_to($log)
+    end
+
+    def try_write(chunk)
+      chunk.write_to($log)
+      commit_write(chunk.unique_id)
+    end
   end
 end

data/lib/fluent/plugin/output.rb

@@ -138,7 +138,7 @@ module Fluent
         end
       end
 
-      attr_reader :as_secondary, :delayed_commit, :delayed_commit_timeout
+      attr_reader :as_secondary, :delayed_commit, :delayed_commit_timeout, :timekey_zone
       attr_reader :num_errors, :emit_count, :emit_records, :write_count, :rollback_count
 
       # for tests
@@ -185,13 +185,21 @@ module Fluent
         @simple_chunking = nil
         @chunk_keys = @chunk_key_time = @chunk_key_tag = nil
         @flush_mode = nil
+        @timekey_zone = nil
       end
 
       def acts_as_secondary(primary)
         @as_secondary = true
         @primary_instance = primary
+        @chunk_keys = @primary_instance.chunk_keys || []
+        @chunk_key_tag = @primary_instance.chunk_key_tag || false
+        if @primary_instance.chunk_key_time
+          @chunk_key_time = @primary_instance.chunk_key_time
+          @timekey_zone = @primary_instance.timekey_zone
+          @output_time_formatter_cache = {}
+        end
+
         (class << self; self; end).module_eval do
-          define_method(:extract_placeholders){ |str, metadata| @primary_instance.extract_placeholders(str, metadata) }
           define_method(:commit_write){ |chunk_id| @primary_instance.commit_write(chunk_id, delayed: delayed_commit, secondary: true) }
           define_method(:rollback_write){ |chunk_id| @primary_instance.rollback_write(chunk_id) }
         end
@@ -251,7 +259,7 @@ module Fluent
           if @chunk_key_time
             raise Fluent::ConfigError, "<buffer ...> argument includes 'time', but timekey is not configured" unless @buffer_config.timekey
             Fluent::Timezone.validate!(@buffer_config.timekey_zone)
-            @buffer_config.timekey_zone = '+0000' if @buffer_config.timekey_use_utc
+            @timekey_zone = @buffer_config.timekey_use_utc ? '+0000' : @buffer_config.timekey_zone
             @output_time_formatter_cache = {}
           end
 
@@ -477,13 +485,13 @@ module Fluent
 
       # TODO: optimize this code
       def extract_placeholders(str, metadata)
-        if metadata.timekey.nil? && metadata.tag.nil? && metadata.variables.nil?
+        if metadata.empty?
           str
         else
           rvalue = str
           # strftime formatting
           if @chunk_key_time # this section MUST be earlier than rest to use raw 'str'
-            @output_time_formatter_cache[str] ||= Fluent::Timezone.formatter(@buffer_config.timekey_zone, str)
+            @output_time_formatter_cache[str] ||= Fluent::Timezone.formatter(@timekey_zone, str)
             rvalue = @output_time_formatter_cache[str].call(metadata.timekey)
           end
           # ${tag}, ${tag[0]}, ${tag[1]}, ...

data/lib/fluent/root_agent.rb

@@ -91,7 +91,7 @@ module Fluent
       else
         conf.elements(name: 'source').each { |e|
           type = e['@type']
-          raise ConfigError, "Missing 'type' parameter on <source> directive" unless type
+          raise ConfigError, "Missing '@type' parameter on <source> directive" unless type
           add_source(type, e)
         }
       end

data/lib/fluent/test/startup_shutdown.rb

@@ -0,0 +1,33 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'serverengine'
+
+module Fluent
+  module Test
+    module StartupShutdown
+      def startup
+        socket_manager_path = ServerEngine::SocketManager::Server.generate_path
+        @server = ServerEngine::SocketManager::Server.open(socket_manager_path)
+        ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
+      end
+
+      def shutdown
+        @server.close
+      end
+    end
+  end
+end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.14.4'
+  VERSION = '0.14.5'
 
 end

data/test/plugin/test_in_forward.rb

@@ -1,47 +1,73 @@
 require_relative '../helper'
 
 require 'fluent/test'
+require 'fluent/test/startup_shutdown'
 require 'base64'
 
 require 'fluent/env'
 require 'fluent/plugin/in_forward'
 
 class ForwardInputTest < Test::Unit::TestCase
-  class << self
-    def startup
-      socket_manager_path = ServerEngine::SocketManager::Server.generate_path
-      @server = ServerEngine::SocketManager::Server.open(socket_manager_path)
-      ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
-    end
-
-    def shutdown
-      @server.close
-    end
-  end
-
   def setup
     Fluent::Test.setup
     @responses = []  # for testing responses after sending data
   end
 
   PORT = unused_port
+
+  SHARED_KEY = 'foobar1'
+  USER_NAME = 'tagomoris'
+  USER_PASSWORD = 'fluentd'
+
   CONFIG = %[
     port #{PORT}
     bind 127.0.0.1
   ]
   PEERADDR = ['?', '0000', '127.0.0.1', '127.0.0.1']
+  CONFIG_AUTH = %[
+    port #{PORT}
+    bind 127.0.0.1
+    <security>
+      self_hostname localhost
+      shared_key foobar1
+      user_auth true
+      <user>
+        username #{USER_NAME}
+        password #{USER_PASSWORD}
+      </user>
+      <client>
+        network 127.0.0.0/8
+        shared_key #{SHARED_KEY}
+        users ["#{USER_NAME}"]
+      </client>
+    </security>
+  ]
 
   def create_driver(conf=CONFIG)
     Fluent::Test::InputTestDriver.new(Fluent::ForwardInput).configure(conf)
   end
 
-  def test_configure
-    d = create_driver
-    assert_equal PORT, d.instance.port
-    assert_equal '127.0.0.1', d.instance.bind
-    assert_equal 0, d.instance.linger_timeout
-    assert_equal 0.5, d.instance.blocking_timeout
-    assert !d.instance.backlog
+  class Configure < self
+    def test_simple
+      d = create_driver
+      assert_equal PORT, d.instance.port
+      assert_equal '127.0.0.1', d.instance.bind
+      assert_equal 0, d.instance.linger_timeout
+      assert_equal 0.5, d.instance.blocking_timeout
+      assert !d.instance.backlog
+    end
+
+    def test_auth
+      d = create_driver(CONFIG_AUTH)
+      assert_equal PORT, d.instance.port
+      assert_equal '127.0.0.1', d.instance.bind
+      assert_equal 0, d.instance.linger_timeout
+      assert !d.instance.backlog
+
+      assert d.instance.security
+      assert_equal 1, d.instance.security.users.size
+      assert_equal 1, d.instance.security.clients.size
+    end
   end
 
   # TODO: Will add Loop::run arity check with stub/mock library
@@ -50,566 +76,1005 @@ class ForwardInputTest < Test::Unit::TestCase
     TCPSocket.new('127.0.0.1', PORT)
   end
 
-  def test_time
-    d = create_driver
+  class Message < self
+    extend Fluent::Test::StartupShutdown
 
-    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
-    Fluent::Engine.now = time
-
-    d.expect_emit "tag1", time, {"a"=>1}
-    d.expect_emit "tag2", time, {"a"=>2}
-
-    d.run do
-      d.expected_emits.each {|tag, _time, record|
-        send_data Fluent::Engine.msgpack_factory.packer.write([tag, 0, record]).to_s
-      }
-    end
-  end
+    def test_time
+      d = create_driver
 
-  def test_message
-    d = create_driver
+      time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+      Fluent::Engine.now = time
 
-    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
-
-    d.expect_emit "tag1", time, {"a"=>1}
-    d.expect_emit "tag2", time, {"a"=>2}
+      records = [
+        ["tag1", time, {"a"=>1}],
+        ["tag2", time, {"a"=>2}],
+      ]
 
-    d.run do
-      d.expected_emits.each {|tag, _time, record|
-        send_data Fluent::Engine.msgpack_factory.packer.write([tag, _time, record]).to_s
-      }
+      d.expected_emits_length = records.length
+      d.run_timeout = 2
+      d.run do
+        records.each {|tag, _time, record|
+          send_data packer.write([tag, 0, record]).to_s
+        }
+      end
+      assert_equal(records, d.emits.sort_by {|a| a[0] })
     end
-  end
 
-  def test_message_with_time_as_integer
-    d = create_driver
+    def test_plain
+      d = create_driver
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+      time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
 
-    d.expect_emit "tag1", time, {"a"=>1}
-    d.expect_emit "tag2", time, {"a"=>2}
+      records = [
+        ["tag1", time, {"a"=>1}],
+        ["tag2", time, {"a"=>2}],
+      ]
 
-    d.run do
-      d.expected_emits.each {|tag, _time, record|
-        send_data Fluent::Engine.msgpack_factory.packer.write([tag, _time, record]).to_s
-      }
+      d.expected_emits_length = records.length
+      d.run_timeout = 2
+      d.run do
+        records.each {|tag, _time, record|
+          send_data packer.write([tag, _time, record]).to_s
+        }
+      end
+      assert_equal(records, d.emits)
     end
-  end
 
-  def test_message_with_skip_invalid_event
-    d = create_driver(CONFIG + "skip_invalid_event true")
+    def test_time_as_integer
+      d = create_driver
 
-    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+      time = Time.parse("2011-01-02 13:14:15 UTC").to_i
 
-    d.expect_emit "tag1", time, {"a" => 1}
-    d.expect_emit "tag2", time, {"a" => 2}
+      records = [
+        ["tag1", time, {"a"=>1}],
+        ["tag2", time, {"a"=>2}],
+      ]
 
-    d.run do
-      entries = d.expected_emits.map {|tag, _time, record| [tag, _time, record] }
-      # These entries are skipped
-      entries << ['tag1', true, {'a' => 3}] << ['tag2', time, 'invalid record']
+      d.expected_emits_length = records.length
+      d.run_timeout = 2
+      d.run do
+        records.each {|tag, _time, record|
+          send_data packer.write([tag, _time, record]).to_s
+        }
+      end
 
-      entries.each {|tag, _time, record|
-        # Without ack, logs are sometimes not saved to logs during test.
-        send_data Fluent::Engine.msgpack_factory.packer.write([tag, _time, record]).to_s, true
-      }
+      assert_equal(records, d.emits)
     end
 
-    assert_equal 2, d.instance.log.logs.count { |line| line =~ /got invalid event and drop it/ }
-  end
-
-  def test_forward
-    d = create_driver
-
-    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+    def test_skip_invalid_event
+      d = create_driver(CONFIG + "skip_invalid_event true")
 
-    d.expect_emit "tag1", time, {"a"=>1}
-    d.expect_emit "tag1", time, {"a"=>2}
-
-    d.run do
-      entries = []
-      d.expected_emits.each {|tag, _time,record|
-        entries << [time, record]
-      }
-      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
-    end
-  end
+      time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
 
-  def test_forward_with_time_as_integer
-    d = create_driver
+      records = [
+        ["tag1", time, {"a" => 1}],
+        ["tag2", time, {"a" => 2}],
+      ]
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+      d.run do
+        entries = records.map { |tag, _time, record| [tag, _time, record] }
+        # These entries are skipped
+        entries << ['tag1', true, {'a' => 3}] << ['tag2', time, 'invalid record']
 
-    d.expect_emit "tag1", time, {"a"=>1}
-    d.expect_emit "tag1", time, {"a"=>2}
+        entries.each {|tag, _time, record|
+          # Without ack, logs are sometimes not saved to logs during test.
+          send_data packer.write([tag, _time, record]).to_s, try_to_receive_response: true
+        }
+      end
 
-    d.run do
-      entries = []
-      d.expected_emits.each {|_tag, _time, record|
-        entries << [_time, record]
-      }
-      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
+      assert_equal 2, d.instance.log.logs.count { |line| line =~ /got invalid event and drop it/ }
+      assert_equal records[0], d.emits[0]
+      assert_equal records[1], d.emits[1]
     end
-  end
 
-  def test_forward_with_skip_invalid_event
-    d = create_driver(CONFIG + "skip_invalid_event true")
+    def test_json
+      d = create_driver
 
-    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+      time = Time.parse("2011-01-02 13:14:15 UTC").to_i
 
-    d.expect_emit "tag1", time, {"a" => 1}
-    d.expect_emit "tag1", time, {"a" => 2}
+      records = [
+        ["tag1", time, {"a"=>1}],
+        ["tag2", time, {"a"=>2}],
+      ]
 
-    d.run do
-      entries = d.expected_emits.map {|_tag, _time, record| [_time, record] }
-      # These entries are skipped
-      entries << ['invalid time', {'a' => 3}] << [time, 'invalid record']
+      d.expected_emits_length = records.length
+      d.run_timeout = 2
+      d.run do
+        records.each {|tag, _time, record|
+          send_data [tag, _time, record].to_json
+        }
+      end
 
-      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
+      assert_equal(records, d.emits.sort_by {|a| a[1] })
     end
 
-    assert_equal 2, d.instance.log.logs.count { |line| line =~ /skip invalid event/ }
-  end
+    def test_json_with_newline
+      d = create_driver
 
-  def test_packed_forward
-    d = create_driver
+      time = Time.parse("2011-01-02 13:14:15 UTC").to_i
 
-    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+      records = [
+        ["tag1", time, {"a"=>1}],
+        ["tag2", time, {"a"=>2}],
+      ]
 
-    d.expect_emit "tag1", time, {"a"=>1}
-    d.expect_emit "tag1", time, {"a"=>2}
+      d.expected_emits_length = records.length
+      d.run_timeout = 2
+      d.run do
+        records.each {|tag, _time, record|
+          send_data [tag, _time, record].to_json + "\n"
+        }
+      end
 
-    d.run do
-      entries = ''
-      d.expected_emits.each {|_tag, _time, record|
-        Fluent::Engine.msgpack_factory.packer(entries).write([_time, record]).flush
-      }
-      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
+      assert_equal(records, d.emits.sort_by {|a| a[1] })
     end
   end
 
-  def test_packed_forward_with_time_as_integer
-    d = create_driver
-
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-
-    d.expect_emit "tag1", time, {"a"=>1}
-    d.expect_emit "tag1", time, {"a"=>2}
+  class Forward < self
+    extend Fluent::Test::StartupShutdown
+
+    data(tcp: {
+           config: CONFIG,
+           options: {
+             auth: false
+           }
+         },
+         auth: {
+           config: CONFIG_AUTH,
+           options: {
+             auth: true
+           }
+         })
+    def test_plain(data)
+      config = data[:config]
+      options = data[:options]
+      d = create_driver(config)
+
+      time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+
+      records = [
+        ["tag1", time, {"a"=>1}],
+        ["tag1", time, {"a"=>2}]
+      ]
+
+      d.expected_emits_length = records.length
+      d.run_timeout = 2
+      d.run do
+        sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
+
+        entries = []
+        records.each {|tag, _time, record|
+          entries << [_time, record]
+        }
+        send_data packer.write(["tag1", entries]).to_s, **options
+      end
+      assert_equal(records, d.emits)
 
-    d.run do
-      entries = ''
-      d.expected_emits.each {|_tag, _time, record|
-        Fluent::Engine.msgpack_factory.packer(entries).write([_time, record]).flush
-      }
-      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
+      sleep 0.1 while d.instance.instance_eval{ @thread }.status # to confirm that plugin stopped completely
     end
-  end
-
-  def test_packed_forward_with_skip_invalid_event
-    d = create_driver(CONFIG + "skip_invalid_event true")
 
-    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
-
-    d.expect_emit "tag1", time, {"a" => 1}
-    d.expect_emit "tag1", time, {"a" => 2}
+    data(tcp: {
+           config: CONFIG,
+           options: {
+             auth: false
+           }
+         },
+         auth: {
+           config: CONFIG_AUTH,
+           options: {
+             auth: true
+           }
+         })
+    def test_time_as_integer(data)
+      config = data[:config]
+      options = data[:options]
+      d = create_driver(config)
+
+      time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+      records = [
+        ["tag1", time, {"a"=>1}],
+        ["tag1", time, {"a"=>2}]
+      ]
+
+      d.expected_emits_length = records.length
+      d.run_timeout = 2
+      d.run do
+        sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
+
+        entries = []
+        records.each {|tag, _time, record|
+          entries << [_time, record]
+        }
+        send_data packer.write(["tag1", entries]).to_s, **options
+      end
 
-    d.run do
-      entries = d.expected_emits.map {|_tag , _time, record| [_time, record] }
-      # These entries are skipped
-      entries << ['invalid time', {'a' => 3}] << [time, 'invalid record']
+      assert_equal(records, d.emits)
 
-      packed_entries = ''
-      entries.each {|_time, record|
-        Fluent::Engine.msgpack_factory.packer(packed_entries).write([_time, record]).flush
-      }
-      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", packed_entries]).to_s
+      sleep 0.1 while d.instance.instance_eval{ @thread }.status # to confirm that plugin stopped completely
     end
 
-    assert_equal 2, d.instance.log.logs.count { |line| line =~ /skip invalid event/ }
-  end
-
-  def test_message_json
-    d = create_driver
-
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    data(tcp: {
+           config: CONFIG,
+           options: {
+             auth: false
+           }
+         },
+         auth: {
+           config: CONFIG_AUTH,
+           options: {
+             auth: true
+           }
+         })
+    def test_skip_invalid_event(data)
+      config = data[:config]
+      options = data[:options]
+      d = create_driver(config + "skip_invalid_event true")
+
+      time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+
+      records = [
+        ["tag1", time, {"a" => 1}],
+        ["tag1", time, {"a" => 2}],
+      ]
+
+      d.expected_emits_length = records.length
+      d.run_timeout = 2
+      d.run do
+        sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
+
+        entries = records.map { |tag, _time, record| [_time, record] }
+        # These entries are skipped
+        entries << ['invalid time', {'a' => 3}] << [time, 'invalid record']
+
+        send_data packer.write(["tag1", entries]).to_s, **options
+      end
 
-    d.expect_emit "tag1", time, {"a"=>1}
-    d.expect_emit "tag2", time, {"a"=>2}
+      assert_equal 2, d.instance.log.logs.count { |line| line =~ /skip invalid event/ }
 
-    d.run do
-      d.expected_emits.each {|tag, _time, record|
-        send_data [tag, _time, record].to_json
-      }
+      sleep 0.1 while d.instance.instance_eval{ @thread }.status # to confirm that plugin stopped completely
     end
   end
 
-  def test_set_size_to_option
-    d = create_driver
+  class PackedForward < self
+    extend Fluent::Test::StartupShutdown
+
+    data(tcp: {
+           config: CONFIG,
+           options: {
+             auth: false
+           }
+         },
+         auth: {
+           config: CONFIG_AUTH,
+           options: {
+             auth: true
+           }
+         })
+    def test_plain(data)
+      config = data[:config]
+      options = data[:options]
+      d = create_driver(config)
+
+      time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+
+      records = [
+        ["tag1", time, {"a"=>1}],
+        ["tag1", time, {"a"=>2}],
+      ]
+
+      d.expected_emits_length = records.length
+      d.run_timeout = 2
+      d.run do
+        sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
+
+        entries = ''
+        records.each {|_tag, _time, record|
+          packer(entries).write([_time, record]).flush
+        }
+        send_data packer.write(["tag1", entries]).to_s, **options
+      end
+      assert_equal(records, d.emits)
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    events = [
-      ["tag1", time, {"a"=>1}],
-      ["tag1", time, {"a"=>2}]
-    ]
+      sleep 0.1 while d.instance.instance_eval{ @thread }.status # to confirm that plugin stopped completely
+    end
 
-    entries = ''
-    events.each {|_tag, _time, record|
-      [_time, record].to_msgpack(entries)
-    }
+    data(tcp: {
+           config: CONFIG,
+           options: {
+             auth: false
+           }
+         },
+         auth: {
+           config: CONFIG_AUTH,
+           options: {
+             auth: true
+           }
+         })
+    def test_time_as_integer(data)
+      config = data[:config]
+      options = data[:options]
+      d = create_driver(config)
+
+      time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+      records = [
+        ["tag1", time, {"a"=>1}],
+        ["tag1", time, {"a"=>2}],
+      ]
+
+      d.expected_emits_length = records.length
+      d.run_timeout = 2
+      d.run do
+        sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
+
+        entries = ''
+        records.each {|tag, _time, record|
+          packer(entries).write([_time, record]).flush
+        }
+        send_data packer.write(["tag1", entries]).to_s, **options
+      end
+      assert_equal(records, d.emits)
 
-    chunk = ["tag1", entries, { 'size' => events.length }].to_msgpack
+      sleep 0.1 while d.instance.instance_eval{ @thread }.status # to confirm that plugin stopped completely
+    end
 
-    d.run do
-      Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
-        option = d.instance.send(:on_message, obj, chunk.size, PEERADDR)
-        assert_equal option['size'], events.length
+    data(tcp: {
+           config: CONFIG,
+           options: {
+             auth: false
+           }
+         },
+         auth: {
+           config: CONFIG_AUTH,
+           options: {
+             auth: true
+           }
+         })
+    def test_skip_invalid_event(data)
+      config = data[:config]
+      options = data[:options]
+      d = create_driver(config + "skip_invalid_event true")
+
+      time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+
+      records = [
+        ["tag1", time, {"a" => 1}],
+        ["tag1", time, {"a" => 2}],
+      ]
+
+      d.expected_emits_length = records.length
+      d.run_timeout = 2
+      d.run do
+        sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
+
+        entries = records.map { |tag, _time, record| [_time, record] }
+        # These entries are skipped
+        entries << ['invalid time', {'a' => 3}] << [time, 'invalid record']
+
+        packed_entries = ''
+        entries.each { |_time, record|
+          packer(packed_entries).write([_time, record]).flush
+        }
+        send_data packer.write(["tag1", packed_entries]).to_s, **options
       end
+
+      assert_equal 2, d.instance.log.logs.count { |line| line =~ /skip invalid event/ }
+
+      sleep 0.1 while d.instance.instance_eval{ @thread }.status # to confirm that plugin stopped completely
     end
   end
 
-  def test_send_large_chunk_warning
-    d = create_driver(CONFIG + %[
+  class Warning < self
+    extend Fluent::Test::StartupShutdown
+
+    def test_send_large_chunk_warning
+      d = create_driver(CONFIG + %[
       chunk_size_warn_limit 16M
       chunk_size_limit 32M
     ])
 
-    time = Fluent::EventTime.parse("2014-04-25 13:14:15 UTC")
+      time = Fluent::EventTime.parse("2014-04-25 13:14:15 UTC")
 
-    # generate over 16M chunk
-    str = "X" * 1024 * 1024
-    chunk = [ "test.tag", (0...16).map{|i| [time + i, {"data" => str}] } ].to_msgpack
-    assert chunk.size > (16 * 1024 * 1024)
-    assert chunk.size < (32 * 1024 * 1024)
+      # generate over 16M chunk
+      str = "X" * 1024 * 1024
+      chunk = [ "test.tag", (0...16).map{|i| [time + i, {"data" => str}] } ].to_msgpack
+      assert chunk.size > (16 * 1024 * 1024)
+      assert chunk.size < (32 * 1024 * 1024)
 
-    d.run do
-      Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
-        d.instance.send(:on_message, obj, chunk.size, PEERADDR)
-      end
-    end
+      d.run do
+        sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
 
-    # check emitted data
-    emits = d.emits
-    assert_equal 16, emits.size
-    assert emits.map(&:first).all?{|t| t == "test.tag" }
-    assert_equal (0...16).to_a, emits.map{|_tag, t, _record| t - time }
+        Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
+          d.instance.send(:on_message, obj, chunk.size, PEERADDR)
+        end
+      end
 
-    # check log
-    assert d.instance.log.logs.select{|line|
-      line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_warn_limit':/ &&
-      line =~ / tag="test.tag" source="host: 127.0.0.1, addr: 127.0.0.1, port: \d+" limit=16777216 size=16777501/
-    }.size == 1, "large chunk warning is not logged"
-  end
+      # check emitted data
+      emits = d.emits
+      assert_equal 16, emits.size
+      assert emits.map(&:first).all?{|t| t == "test.tag" }
+      assert_equal (0...16).to_a, emits.map{|_tag, t, _record| t - time }
+
+      # check log
+      assert d.instance.log.logs.select{|line|
+        line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_warn_limit':/ &&
+        line =~ / tag="test.tag" source="host: 127.0.0.1, addr: 127.0.0.1, port: \d+" limit=16777216 size=16777501/
+      }.size == 1, "large chunk warning is not logged"
+    end
 
-  def test_send_large_chunk_only_warning
-    d = create_driver(CONFIG + %[
+    def test_send_large_chunk_only_warning
+      d = create_driver(CONFIG + %[
       chunk_size_warn_limit 16M
     ])
-    time = Fluent::EventTime.parse("2014-04-25 13:14:15 UTC")
+      time = Fluent::EventTime.parse("2014-04-25 13:14:15 UTC")
 
-    # generate over 16M chunk
-    str = "X" * 1024 * 1024
-    chunk = [ "test.tag", (0...16).map{|i| [time + i, {"data" => str}] } ].to_msgpack
+      # generate over 16M chunk
+      str = "X" * 1024 * 1024
+      chunk = [ "test.tag", (0...16).map{|i| [time + i, {"data" => str}] } ].to_msgpack
 
-    d.run do
-      Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
-        d.instance.send(:on_message, obj, chunk.size, PEERADDR)
+      d.run do
+        sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
+
+        Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
+          d.instance.send(:on_message, obj, chunk.size, PEERADDR)
+        end
       end
-    end
 
-    # check log
-    assert d.instance.log.logs.select{ |line|
-      line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_warn_limit':/ &&
-      line =~ / tag="test.tag" source="host: 127.0.0.1, addr: 127.0.0.1, port: \d+" limit=16777216 size=16777501/
-    }.size == 1, "large chunk warning is not logged"
-  end
+      # check log
+      assert d.instance.log.logs.select{ |line|
+        line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_warn_limit':/ &&
+        line =~ / tag="test.tag" source="host: 127.0.0.1, addr: 127.0.0.1, port: \d+" limit=16777216 size=16777501/
+      }.size == 1, "large chunk warning is not logged"
+    end
 
-  def test_send_large_chunk_limit
-    d = create_driver(CONFIG + %[
+    def test_send_large_chunk_limit
+      d = create_driver(CONFIG + %[
       chunk_size_warn_limit 16M
       chunk_size_limit 32M
     ])
 
-    time = Fluent::EventTime.parse("2014-04-25 13:14:15 UTC")
+      time = Fluent::EventTime.parse("2014-04-25 13:14:15 UTC")
 
-    # generate over 32M chunk
-    str = "X" * 1024 * 1024
-    chunk = [ "test.tag", (0...32).map{|i| [time + i, {"data" => str}] } ].to_msgpack
-    assert chunk.size > (32 * 1024 * 1024)
+      # generate over 32M chunk
+      str = "X" * 1024 * 1024
+      chunk = [ "test.tag", (0...32).map{|i| [time + i, {"data" => str}] } ].to_msgpack
+      assert chunk.size > (32 * 1024 * 1024)
 
-    # d.run => send_data
-    d.run do
-      Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
-        d.instance.send(:on_message, obj, chunk.size, PEERADDR)
-      end
-    end
-
-    # check emitted data
-    emits = d.emits
-    assert_equal 0, emits.size
+      # d.run => send_data
+      d.run do
+        sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
 
-    # check log
-    assert d.instance.log.logs.select{|line|
-      line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_limit', dropped:/ &&
-      line =~ / tag="test.tag" source="host: 127.0.0.1, addr: 127.0.0.1, port: \d+" limit=33554432 size=33554989/
-    }.size == 1, "large chunk warning is not logged"
-  end
+        Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
+          d.instance.send(:on_message, obj, chunk.size, PEERADDR)
+        end
+      end
 
-  data('string chunk' => 'broken string',
-       'integer chunk' => 10)
-  def test_send_broken_chunk(data)
-    d = create_driver
+      # check emitted data
+      emits = d.emits
+      assert_equal 0, emits.size
 
-    # d.run => send_data
-    d.run do
-      d.instance.send(:on_message, data, 1000000000, PEERADDR)
+      # check log
+      assert d.instance.log.logs.select{|line|
+        line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_limit', dropped:/ &&
+        line =~ / tag="test.tag" source="host: 127.0.0.1, addr: 127.0.0.1, port: \d+" limit=33554432 size=33554989/
+      }.size == 1, "large chunk warning is not logged"
     end
 
-    # check emitted data
-    emits = d.emits
-    assert_equal 0, emits.size
-
-    # check log
-    assert d.instance.log.logs.select{|line|
-      line =~ / \[warn\]: incoming chunk is broken: source="host: 127.0.0.1, addr: 127.0.0.1, port: \d+" msg=#{data.inspect}/
-    }.size == 1, "should not accept broken chunk"
-  end
-
-  def test_respond_to_message_requiring_ack
-    d = create_driver
+    data('string chunk' => 'broken string',
+         'integer chunk' => 10)
+    def test_send_broken_chunk(data)
+      d = create_driver
 
-    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+      # d.run => send_data
+      d.run do
+        sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
 
-    events = [
-      ["tag1", time, {"a"=>1}],
-      ["tag2", time, {"a"=>2}]
-    ]
-    d.expected_emits_length = events.length
+        d.instance.send(:on_message, data, 1000000000, PEERADDR)
+      end
 
-    expected_acks = []
+      # check emitted data
+      emits = d.emits
+      assert_equal 0, emits.size
 
-    d.run do
-      events.each {|tag, _time, record|
-        op = { 'chunk' => Base64.encode64(record.object_id.to_s) }
-        expected_acks << op['chunk']
-        send_data [tag, _time, record, op].to_msgpack, true
-      }
+      # check log
+      assert d.instance.log.logs.select{|line|
+        line =~ / \[warn\]: incoming chunk is broken: source="host: 127.0.0.1, addr: 127.0.0.1, port: \d+" msg=#{data.inspect}/
+      }.size == 1, "should not accept broken chunk"
     end
-
-    assert_equal events, d.emits
-    assert_equal expected_acks, @responses.map { |res| MessagePack.unpack(res)['ack'] }
   end
 
-  # FIX: response is not pushed into @responses because IO.select has been blocked until InputForward shutdowns
-  def test_respond_to_forward_requiring_ack
-    d = create_driver
-
-    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
-
-    events = [
-      ["tag1", time, {"a"=>1}],
-      ["tag1", time, {"a"=>2}]
-    ]
-    d.expected_emits_length = events.length
+  class RespondToRequiringAck < self
+    extend Fluent::Test::StartupShutdown
+
+    data(tcp: {
+           config: CONFIG,
+           options: {
+             auth: false
+           }
+         },
+         auth: {
+           config: CONFIG_AUTH,
+           options: {
+             auth: true
+           }
+         })
+    def test_message(data)
+      config = data[:config]
+      options = data[:options]
+      d = create_driver(config)
+
+      time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+
+      events = [
+        ["tag1", time, {"a"=>1}],
+        ["tag2", time, {"a"=>2}]
+      ]
+      d.expected_emits_length = events.length
+
+      expected_acks = []
+
+      d.run do
+        sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
+
+        events.each {|tag, _time, record|
+          op = { 'chunk' => Base64.encode64(record.object_id.to_s) }
+          expected_acks << op['chunk']
+          send_data [tag, _time, record, op].to_msgpack, try_to_receive_response: true, **options
+        }
+      end
 
-    expected_acks = []
+      assert_equal events, d.emits
+      assert_equal expected_acks, @responses.map { |res| MessagePack.unpack(res)['ack'] }
 
-    d.run do
-      entries = []
-      events.each {|_tag, _time, record|
-        entries << [time, record]
-      }
-      op = { 'chunk' => Base64.encode64(entries.object_id.to_s) }
-      expected_acks << op['chunk']
-      send_data ["tag1", entries, op].to_msgpack, true
+      sleep 0.1 while d.instance.instance_eval{ @thread }.status # to confirm that plugin stopped completely
     end
 
-    assert_equal events, d.emits
-    assert_equal expected_acks, @responses.map { |res| MessagePack.unpack(res)['ack'] }
-  end
-
-  def test_respond_to_packed_forward_requiring_ack
-    d = create_driver
-
-    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
-
-    events = [
-      ["tag1", time, {"a"=>1}],
-      ["tag1", time, {"a"=>2}]
-    ]
-    d.expected_emits_length = events.length
+    # FIX: response is not pushed into @responses because IO.select has been blocked until InputForward shutdowns
+    data(tcp: {
+           config: CONFIG,
+           options: {
+             auth: false
+           }
+         },
+         auth: {
+           config: CONFIG_AUTH,
+           options: {
+             auth: true
+           }
+         })
+    def test_forward(data)
+      config = data[:config]
+      options = data[:options]
+      d = create_driver(config)
+
+      time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+
+      events = [
+        ["tag1", time, {"a"=>1}],
+        ["tag1", time, {"a"=>2}]
+      ]
+      d.expected_emits_length = events.length
+
+      expected_acks = []
+
+      d.run do
+        sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
+
+        entries = []
+        events.each {|_tag, _time, record|
+          entries << [_time, record]
+        }
+        op = { 'chunk' => Base64.encode64(entries.object_id.to_s) }
+        expected_acks << op['chunk']
+        send_data ["tag1", entries, op].to_msgpack, try_to_receive_response: true, **options
+      end
 
-    expected_acks = []
+      assert_equal events, d.emits
+      assert_equal expected_acks, @responses.map { |res| MessagePack.unpack(res)['ack'] }
 
-    d.run do
-      entries = ''
-      events.each {|_tag, _time, record|
-        [_time, record].to_msgpack(entries)
-      }
-      op = { 'chunk' => Base64.encode64(entries.object_id.to_s) }
-      expected_acks << op['chunk']
-      send_data ["tag1", entries, op].to_msgpack, true
+      sleep 0.1 while d.instance.instance_eval{ @thread }.status # to confirm that plugin stopped completely
     end
 
-    assert_equal events, d.emits
-    assert_equal expected_acks, @responses.map { |res| MessagePack.unpack(res)['ack'] }
-  end
+    data(tcp: {
+           config: CONFIG,
+           options: {
+             auth: false
+           }
+         },
+         auth: {
+           config: CONFIG_AUTH,
+           options: {
+             auth: true
+           }
+         })
+    def test_packed_forward
+      config = data[:config]
+      options = data[:options]
+      d = create_driver(config)
+
+      time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+
+      events = [
+        ["tag1", time, {"a"=>1}],
+        ["tag1", time, {"a"=>2}]
+      ]
+      d.expected_emits_length = events.length
+
+      expected_acks = []
+
+      d.run do
+        sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
+
+        entries = ''
+        events.each {|_tag, _time,record|
+          [time, record].to_msgpack(entries)
+        }
+        op = { 'chunk' => Base64.encode64(entries.object_id.to_s) }
+        expected_acks << op['chunk']
+        send_data ["tag1", entries, op].to_msgpack, try_to_receive_response: true, **options
+      end
 
-  def test_respond_to_message_json_requiring_ack
-    d = create_driver
+      assert_equal events, d.emits
+      assert_equal expected_acks, @responses.map { |res| MessagePack.unpack(res)['ack'] }
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+      sleep 0.1 while d.instance.instance_eval{ @thread }.status # to confirm that plugin stopped completely
+    end
 
-    events = [
-      ["tag1", time, {"a"=>1}],
-      ["tag2", time, {"a"=>2}]
-    ]
-    d.expected_emits_length = events.length
+    data(tcp: {
+           config: CONFIG,
+           options: {
+             auth: false
+           }
+         },
+         auth: {
+           config: CONFIG_AUTH,
+           options: {
+             auth: true
+           }
+         })
+    def test_message_json(data)
+      config = data[:config]
+      options = data[:options]
+      omit "with json, auth doen NOT work" if options[:auth]
+      d = create_driver(config)
+
+      time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+      events = [
+        ["tag1", time, {"a"=>1}],
+        ["tag2", time, {"a"=>2}]
+      ]
+      d.expected_emits_length = events.length
+
+      expected_acks = []
+
+      d.expected_emits_length = events.length
+      d.run_timeout = 2
+      d.run do
+        sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
+
+        events.each {|tag, _time, record|
+          op = { 'chunk' => Base64.encode64(record.object_id.to_s) }
+          expected_acks << op['chunk']
+          send_data [tag, _time, record, op].to_json, try_to_receive_response: true, **options
+        }
+      end
 
-    expected_acks = []
+      assert_equal events, d.emits
+      assert_equal expected_acks, @responses.map { |res| JSON.parse(res)['ack'] }
 
-    d.run do
-      events.each {|tag, _time, record|
-        op = { 'chunk' => Base64.encode64(record.object_id.to_s) }
-        expected_acks << op['chunk']
-        send_data [tag, _time, record, op].to_json, true
-      }
+      sleep 0.1 while d.instance.instance_eval{ @thread }.status # to confirm that plugin stopped completely
     end
-
-    assert_equal events, d.emits
-    assert_equal expected_acks, @responses.map { |res| JSON.parse(res)['ack'] }
   end
 
-  def test_not_respond_to_message_not_requiring_ack
-    d = create_driver
-
-    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+  class NotRespondToNotRequiringAck < self
+    extend Fluent::Test::StartupShutdown
+
+    data(tcp: {
+           config: CONFIG,
+           options: {
+             auth: false
+           }
+         },
+         auth: {
+           config: CONFIG_AUTH,
+           options: {
+             auth: true
+           }
+         })
+    def test_message
+      config = data[:config]
+      options = data[:options]
+      d = create_driver(config)
+
+      time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+
+      events = [
+        ["tag1", time, {"a"=>1}],
+        ["tag2", time, {"a"=>2}]
+      ]
+      d.expected_emits_length = events.length
+
+      d.run do
+        sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
+
+        events.each {|tag, _time, record|
+          send_data [tag, _time, record].to_msgpack, try_to_receive_response: true, **options
+        }
+      end
 
-    events = [
-      ["tag1", time, {"a"=>1}],
-      ["tag2", time, {"a"=>2}]
-    ]
-    d.expected_emits_length = events.length
+      assert_equal events, d.emits
+      assert_equal ["", ""], @responses
 
-    d.run do
-      events.each {|tag, _time, record|
-        send_data [tag, _time, record].to_msgpack, true
-      }
+      sleep 0.1 while d.instance.instance_eval{ @thread }.status # to confirm that plugin stopped completely
     end
 
-    assert_equal events, d.emits
-    assert_equal [nil, nil], @responses
-  end
-
-  def test_not_respond_to_forward_not_requiring_ack
-    d = create_driver
-
-    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+    data(tcp: {
+           config: CONFIG,
+           options: {
+             auth: false
+           }
+         },
+         auth: {
+           config: CONFIG_AUTH,
+           options: {
+             auth: true
+           }
+         })
+    def test_forward(data)
+      config = data[:config]
+      options = data[:options]
+      d = create_driver(config)
+
+      time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+
+      events = [
+        ["tag1", time, {"a"=>1}],
+        ["tag1", time, {"a"=>2}]
+      ]
+      d.expected_emits_length = events.length
+
+      d.run do
+        sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
+
+        entries = []
+        events.each {|tag, _time, record|
+          entries << [_time, record]
+        }
+        send_data ["tag1", entries].to_msgpack, try_to_receive_response: true, **options
+      end
 
-    events = [
-      ["tag1", time, {"a"=>1}],
-      ["tag1", time, {"a"=>2}]
-    ]
-    d.expected_emits_length = events.length
+      assert_equal events, d.emits
+      assert_equal [nil], @responses
 
-    d.run do
-      entries = []
-      events.each {|_tag, _time, record|
-        entries << [_time, record]
-      }
-      send_data ["tag1", entries].to_msgpack, true
+      sleep 0.1 while d.instance.instance_eval{ @thread }.status # to confirm that plugin stopped completely
     end
 
-    assert_equal events, d.emits
-    assert_equal [nil], @responses
-  end
+    data(tcp: {
+           config: CONFIG,
+           options: {
+             auth: false
+           }
+         },
+         auth: {
+           config: CONFIG_AUTH,
+           options: {
+             auth: true
+           }
+         })
+    def test_packed_forward(data)
+      config = data[:config]
+      options = data[:options]
+      d = create_driver(config)
+
+      time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+
+      events = [
+        ["tag1", time, {"a"=>1}],
+        ["tag1", time, {"a"=>2}]
+      ]
+      d.expected_emits_length = events.length
+
+      d.run do
+        sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
+
+        entries = ''
+        events.each {|tag, _time, record|
+          [_time, record].to_msgpack(entries)
+        }
+        send_data ["tag1", entries].to_msgpack, try_to_receive_response: true, **options
+      end
 
-  def test_not_respond_to_packed_forward_not_requiring_ack
-    d = create_driver
+      assert_equal events, d.emits
+      assert_equal [nil], @responses
 
-    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+      sleep 0.1 while d.instance.instance_eval{ @thread }.status # to confirm that plugin stopped completely
+    end
 
-    events = [
-      ["tag1", time, {"a"=>1}],
-      ["tag1", time, {"a"=>2}]
-    ]
-    d.expected_emits_length = events.length
+    data(tcp: {
+           config: CONFIG,
+           options: {
+             auth: false
+           }
+         },
+         auth: {
+           config: CONFIG_AUTH,
+           options: {
+             auth: true
+           }
+         })
+    def test_message_json(data)
+      config = data[:config]
+      options = data[:options]
+      omit "with json, auth doen NOT work" if options[:auth]
+      d = create_driver(config)
+
+      time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+      events = [
+        ["tag1", time, {"a"=>1}],
+        ["tag2", time, {"a"=>2}]
+      ]
+      d.expected_emits_length = events.length
+
+      d.run do
+        sleep 0.1 until d.instance.instance_eval{ @thread } && d.instance.instance_eval{ @thread }.status
+
+        events.each {|tag, _time, record|
+          send_data [tag, _time, record].to_json, try_to_receive_response: true, **options
+        }
+      end
 
-    d.run do
-      entries = ''
-      events.each {|_tag, _time, record|
-        [_time, record].to_msgpack(entries)
-      }
-      send_data ["tag1", entries].to_msgpack, true
+      assert_equal events, d.emits
+      assert_equal [nil, nil], @responses
+
+      sleep 0.1 while d.instance.instance_eval{ @thread }.status # to confirm that plugin stopped completely
     end
+  end
 
-    assert_equal events, d.emits
-    assert_equal [nil], @responses
+  def packer(*args)
+    Fluent::Engine.msgpack_factory.packer(*args)
   end
 
-  def test_not_respond_to_message_json_not_requiring_ack
-    d = create_driver
+  def unpacker
+    Fluent::Engine.msgpack_factory.unpacker
+  end
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+  # res
+  # '' : socket is disconnected without any data
+  # nil: socket read timeout
+  def read_data(io, timeout, &block)
+    res = ''
+    select_timeout = 2
+    timeout_at = Time.now + timeout
+    begin
+      buf = ''
+      io_activated = false
+      while Time.now < timeout_at
+        if IO.select([io], nil, nil, select_timeout)
+          io_activated = true
+          buf = io.readpartial(2048)
+          res ||= ''
+          res << buf
+
+          break if block.call(res)
+        end
+      end
+      res = nil unless io_activated # timeout without no data arrival
+    rescue Errno::EAGAIN
+      sleep 0.01
+      retry if res == ''
+      # if res is not empty, all data in socket buffer are read, so do not retry
+    rescue IOError, EOFError, Errno::ECONNRESET
+      # socket disconnected
+    end
+    res
+  end
 
-    events = [
-      ["tag1", time, {"a"=>1}],
-      ["tag2", time, {"a"=>2}]
+  def simulate_auth_sequence(io, shared_key=SHARED_KEY, username=USER_NAME, password=USER_PASSWORD)
+    auth_response_timeout = 30
+    shared_key_salt = 'salt'
+
+    # reading helo
+    helo_data = read_data(io, auth_response_timeout){|data| MessagePack.unpack(data) rescue nil }
+    raise "Authentication packet timeout" unless helo_data
+    raise "Authentication connection closed" if helo_data == ''
+    # ['HELO', options(hash)]
+    helo = MessagePack.unpack(helo_data)
+    raise "Invalid HELO header" unless helo[0] == 'HELO'
+    raise "Invalid HELO option object" unless helo[1].is_a?(Hash)
+    @options = helo[1]
+
+    # sending ping
+    ping = [
+      'PING',
+      'selfhostname',
+      shared_key_salt,
+      Digest::SHA512.new
+        .update(shared_key_salt)
+        .update('selfhostname')
+        .update(@options['nonce'])
+        .update(shared_key).hexdigest,
     ]
-    d.expected_emits_length = events.length
-
-    d.run do
-      events.each {|tag, _time, record|
-        send_data [tag, _time, record].to_json, true
-      }
+    if @options['auth'] # auth enabled -> value is auth salt
+      pass_digest = Digest::SHA512.new.update(@options['auth']).update(username).update(password).hexdigest
+      ping.push(username, pass_digest)
+    else
+      ping.push('', '')
     end
-
-    assert_equal events, d.emits
-    assert_equal [nil, nil], @responses
+    io.write ping.to_msgpack
+    io.flush
+
+    # reading pong
+    pong_data = read_data(io, auth_response_timeout){|data| MessagePack.unpack(data) rescue nil }
+    raise "PONG packet timeout" unless pong_data
+    raise "PONG connection closed" if pong_data == ''
+    # ['PING', bool(auth_result), string(reason_if_failed), self_hostname, shared_key_digest]
+    pong = MessagePack.unpack(pong_data)
+    raise "Invalid PONG header" unless pong[0] == 'PONG'
+    raise "Authentication Failure: #{pong[2]}" unless pong[1]
+    clientside_calculated = Digest::SHA512.new
+      .update(shared_key_salt)
+      .update(pong[3])
+      .update(@options['nonce'])
+      .update(shared_key).hexdigest
+    raise "Shared key digest mismatch" unless clientside_calculated == pong[4]
+
+    # authentication success
+    true
   end
 
-  def send_data(data, try_to_receive_response=false, response_timeout=1)
+  # Data ordering is not assured:
+  #  Records in different sockets are processed on different thread, so its scheduling make effect
+  #  on order of emitted records.
+  #  So, we MUST sort emitted records in different `send_data` before assertion.
+  def send_data(data, try_to_receive_response: false, response_timeout: 5, auth: false)
     io = connect
-    begin
-      io.write data
-      if try_to_receive_response
-        if IO.select([io], nil, nil, response_timeout)
-          res = io.recv(1024)
-        end
-        # timeout means no response, so push nil to @responses
-      end
-    ensure
-      io.close
+
+    if auth
+      simulate_auth_sequence(io)
+    end
+
+    io.write data
+    io.flush
+    if try_to_receive_response
+      @responses << read_data(io, response_timeout){|d| MessagePack.unpack(d) rescue nil }
     end
-    @responses << res if try_to_receive_response
+  ensure
+    io.close rescue nil # SSL socket requires any writes to close sockets
   end
 
-  # TODO: Use sub_test_case. Currently Errno::EADDRINUSE happens inside sub_test_case
-  test 'message protocol with source_hostname_key' do
-    execute_test { |events|
-      events.each { |tag, time, record|
-        send_data [tag, time, record].to_msgpack
+  sub_test_case 'source_hostname_key feature' do
+    extend Fluent::Test::StartupShutdown
+
+    test 'message protocol with source_hostname_key' do
+      execute_test { |events|
+        events.each { |tag, time, record|
+          send_data [tag, time, record].to_msgpack
+        }
       }
-    }
-  end
+    end
 
-  test 'forward protocol with source_hostname_key' do
-    execute_test { |events|
-      entries = []
-      events.each {|tag,time,record|
-        entries << [time, record]
+    test 'forward protocol with source_hostname_key' do
+      execute_test { |events|
+        entries = []
+        events.each {|tag,time,record|
+          entries << [time, record]
+        }
+        send_data ['tag1', entries].to_msgpack
       }
-      send_data ['tag1', entries].to_msgpack
-    }
-  end
+    end
 
-  test 'packed forward protocol with source_hostname_key' do
-    execute_test { |events|
-      entries = ''
-      events.each { |tag, time, record|
-        Fluent::Engine.msgpack_factory.packer(entries).write([time, record]).flush
+    test 'packed forward protocol with source_hostname_key' do
+      execute_test { |events|
+        entries = ''
+        events.each { |tag, time, record|
+          Fluent::Engine.msgpack_factory.packer(entries).write([time, record]).flush
+        }
+        send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
       }
-      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
-    }
+    end
   end
 
   def execute_test(&block)