fluentd 1.12.0-x86-mingw32 → 1.13.0-x86-mingw32

data/test/command/test_plugin_config_formatter.rb

@@ -225,6 +225,30 @@ TEXT
 path to something
 
 
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+    test "input simple (table)" do
+      dumped_config = capture_stdout do
+        FluentPluginConfigFormatter.new(["--format=markdown", "--table", "input", "simple"]).call
+      end
+      expected = <<TEXT
+## Plugin helpers
+
+* [inject](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-inject)
+* [compat_parameters](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-compat_parameters)
+
+* See also: [Input Plugin Overview](https://docs.fluentd.org/v/1.0/input#overview)
+
+## TestFluentPluginConfigFormatter::SimpleInput
+
+### Configuration
+
+|parameter|type|description|default|
+|---|---|---|---|
+|path|string (required)|path to something||
+
 TEXT
       assert_equal(expected, dumped_config)
     end
@@ -298,6 +322,49 @@ Default value: `normal`.
 
 
 
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+    test "output complex (table)" do
+      dumped_config = capture_stdout do
+        FluentPluginConfigFormatter.new(["--format=markdown", "--table", "output", "complex"]).call
+      end
+      expected = <<TEXT
+## Plugin helpers
+
+* [inject](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-inject)
+* [compat_parameters](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-compat_parameters)
+
+* See also: [Output Plugin Overview](https://docs.fluentd.org/v/1.0/output#overview)
+
+## TestFluentPluginConfigFormatter::ComplexOutput
+
+
+### \\<authentication\\> section (required) (single)
+
+### Configuration
+
+|parameter|type|description|default|
+|---|---|---|---|
+|username|string (required)|username||
+|password|string (required)|password||
+
+
+### \\<parent\\> section (optional) (multiple)
+
+
+#### \\<child\\> section (optional) (multiple)
+
+### Configuration
+
+|parameter|type|description|default|
+|---|---|---|---|
+|names|array (required)|names||
+|difficulty|enum (optional)|difficulty (`easy`, `normal`, `hard`)|`normal`|
+
+
+
 TEXT
       assert_equal(expected, dumped_config)
     end

data/test/config/test_configurable.rb

@@ -1453,7 +1453,7 @@ module Fluent::Config
         @example = ConfigurableSpec::ExampleWithSkipAccessor.new
         @example.configure(config_element('ROOT'))
         assert_equal 'example7', @example.instance_variable_get(:@name)
-        assert_raise NoMethodError.new("undefined method `name' for #{@example}") do
+        assert_raise NoMethodError do
           @example.name
         end
       end

data/test/config/test_system_config.rb

@@ -143,5 +143,51 @@ module Fluent::Config
       sc.overwrite_variables(**s.for_system_config)
       assert_equal(level, sc.log_level)
     end
+
+    sub_test_case "log rotation" do
+      data('daily' => "daily",
+           'weekly' => 'weekly',
+           'monthly' => 'monthly')
+      test "symbols for rotate_age" do |age|
+        conf = parse_text(<<-EOS)
+          <system>
+            <log>
+              rotate_age #{age}
+            </log>
+          </system>
+        EOS
+        sc = Fluent::SystemConfig.new(conf)
+        assert_equal(age.to_sym, sc.log.rotate_age)
+      end
+
+      test "numeric number for rotate age" do
+        conf = parse_text(<<-EOS)
+          <system>
+            <log>
+              rotate_age 3
+            </log>
+          </system>
+        EOS
+        s = FakeSupervisor.new
+        sc = Fluent::SystemConfig.new(conf)
+        assert_equal(3, sc.log.rotate_age)
+      end
+
+      data(h: ['100', 100],
+           k: ['1k', 1024],
+           m: ['1m', 1024 * 1024],
+           g: ['1g', 1024 * 1024 * 1024])
+      test "numeric and SI prefix for rotate_size" do |(label, size)|
+        conf = parse_text(<<-EOS)
+          <system>
+            <log>
+              rotate_size #{label}
+            </log>
+          </system>
+        EOS
+        sc = Fluent::SystemConfig.new(conf)
+        assert_equal(size, sc.log.rotate_size)
+      end
+    end
   end
 end

data/test/plugin/in_tail/test_io_handler.rb

@@ -30,7 +30,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
     end
 
     returned_lines = ''
-    r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, log: $log, open_on_every_update: false) do |lines, _watcher|
+    r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false) do |lines, _watcher|
       returned_lines << lines.join
       true
     end
@@ -62,7 +62,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
       end
 
       returned_lines = ''
-      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, log: $log, open_on_every_update: true) do |lines, _watcher|
+      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: true) do |lines, _watcher|
         returned_lines << lines.join
         true
       end
@@ -93,7 +93,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
       end
 
       returned_lines = []
-      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, log: $log, open_on_every_update: false) do |lines, _watcher|
+      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false) do |lines, _watcher|
         returned_lines << lines.dup
         true
       end
@@ -119,7 +119,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
       end
 
       returned_lines = []
-      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, log: $log, open_on_every_update: false) do |lines, _watcher|
+      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false) do |lines, _watcher|
         returned_lines << lines.dup
         true
       end

data/test/plugin/in_tail/test_position_file.rb

@@ -85,7 +85,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
       assert_equal 5, lines.size
     end
 
-    test 'update seek postion of remained position entry' do
+    test 'update seek position of remained position entry' do
       pf = Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log})
       target_info1 = Fluent::Plugin::TailInput::TargetInfo.new('path1', -1)
       target_info2 = Fluent::Plugin::TailInput::TargetInfo.new('path2', -1)
@@ -147,7 +147,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
       write_data(@file, TEST_CONTENT)
       pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
 
-      valid_target_info = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', Fluent::FileWrapper.stat(@file).ino)
+      valid_target_info = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', File.stat(@file).ino)
       f = pf[valid_target_info]
       assert_equal Fluent::Plugin::TailInput::FilePositionEntry, f.class
       assert_equal 2, f.read_pos
@@ -177,7 +177,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
       assert_equal 0, f.read_inode
       assert_equal 0, f.read_pos
 
-      pf[Fluent::Plugin::TailInput::TargetInfo.new('valid_path', Fluent::FileWrapper.stat(@file).ino)].update(1, 2)
+      pf[Fluent::Plugin::TailInput::TargetInfo.new('valid_path', File.stat(@file).ino)].update(1, 2)
 
       f = pf[Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)]
       assert_equal 0, f.read_inode
@@ -193,7 +193,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
     test 'deletes entry by path' do
       write_data(@file, TEST_CONTENT)
       pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, logger: $log)
-      inode1 = Fluent::FileWrapper.stat(@file).ino
+      inode1 = File.stat(@file).ino
       target_info1 = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', inode1)
       p1 = pf[target_info1]
       assert_equal Fluent::Plugin::TailInput::FilePositionEntry, p1.class
@@ -201,7 +201,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
       pf.unwatch(target_info1)
       assert_equal p1.read_pos, Fluent::Plugin::TailInput::PositionFile::UNWATCHED_POSITION
 
-      inode2 = Fluent::FileWrapper.stat(@file).ino
+      inode2 = File.stat(@file).ino
       target_info2 = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', inode2)
       p2 = pf[target_info2]
       assert_equal Fluent::Plugin::TailInput::FilePositionEntry, p2.class
@@ -282,4 +282,58 @@ class IntailPositionFileTest < Test::Unit::TestCase
       assert_equal 2, f.read_inode
     end
   end
+
+  sub_test_case "TargetInfo equality rules" do
+    sub_test_case "== operator" do
+      def test_equal
+        t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
+        t2 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1235)
+
+        assert_equal t1, t2
+      end
+
+      def test_not_equal
+        t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
+        t2 = Fluent::Plugin::TailInput::TargetInfo.new("test2", 1234)
+
+        assert_not_equal t1, t2
+      end
+    end
+
+    sub_test_case "eql? method" do
+      def test_eql?
+        t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
+        t2 = Fluent::Plugin::TailInput::TargetInfo.new("test", 5321)
+
+        assert do
+          t1.eql? t2
+        end
+      end
+
+      def test_not_eql?
+        t1 = Fluent::Plugin::TailInput::TargetInfo.new("test2", 1234)
+        t2 = Fluent::Plugin::TailInput::TargetInfo.new("test3", 1234)
+
+        assert do
+          !t1.eql? t2
+        end
+      end
+    end
+
+    sub_test_case "hash" do
+      def test_equal
+        t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
+        t2 = Fluent::Plugin::TailInput::TargetInfo.new("test", 7321)
+
+        assert_equal t1.hash, t2.hash
+      end
+
+      def test_not_equal
+        t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
+        t2 = Fluent::Plugin::TailInput::TargetInfo.new("test2", 1234)
+
+        assert_not_equal t1.hash, t2.hash
+      end
+    end
+  end
 end

data/test/plugin/test_file_wrapper.rb

@@ -0,0 +1,115 @@
+require_relative '../helper'
+require 'fluent/plugin/file_wrapper'
+
+class FileWrapperTest < Test::Unit::TestCase
+  require 'windows/file'
+  require 'windows/error'
+  include Windows::File
+  include Windows::Error
+
+  TMP_DIR = File.dirname(__FILE__) + "/../tmp/file_wrapper#{ENV['TEST_ENV_NUMBER']}"
+
+  def setup
+    FileUtils.mkdir_p(TMP_DIR)
+  end
+
+  def teardown
+    FileUtils.rm_rf(TMP_DIR)
+  end
+
+  sub_test_case 'Win32Error' do
+    test 'equal' do
+      assert_equal(Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "message"),
+                   Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "message"))
+    end
+
+    test 'different error code' do
+      assert_not_equal(Fluent::Win32Error.new(ERROR_FILE_NOT_FOUND),
+                       Fluent::Win32Error.new(ERROR_SHARING_VIOLATION))
+    end
+
+    test 'different error message' do
+      assert_not_equal(Fluent::Win32Error.new(ERROR_FILE_NOT_FOUND, "message1"),
+                       Fluent::Win32Error.new(ERROR_FILE_NOT_FOUND, "message2"))
+    end
+
+    test 'different class' do
+      assert_not_equal(Errno::EPIPE,
+                       Fluent::Win32Error.new(ERROR_SHARING_VIOLATION))
+    end
+
+    test 'ERROR_SHARING_VIOLATION message' do
+      assert_equal(Fluent::Win32Error.new(ERROR_SHARING_VIOLATION).message,
+                   "Fluent::Win32Error: code: 32, The process cannot access the file because it is being used by another process.")
+    end
+
+    test 'ERROR_SHARING_VIOLATION with a message' do
+      assert_equal(Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "cannot open the file").message,
+                   "Fluent::Win32Error: code: 32, The process cannot access the file because it is being used by another process." +
+                   " - cannot open the file")
+    end
+
+    test 'to_s' do
+      assert_equal("Fluent::Win32Error: code: 32, The process cannot access the file because it is being used by another process. - C:\file.txt",
+                   Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "C:\file.txt").to_s)
+    end
+
+    test 'inspect' do
+      assert_equal("#<Fluent::Win32Error: code: 32, The process cannot access the file because it is being used by another process. - C:\file.txt>",
+                   Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "C:\file.txt").inspect)
+    end
+
+    data('0' => [false, 0],
+         '9999' => [false, 9999],
+         '10000' => [true, 10000],
+         '10001' => [true, 10001])
+    test 'wsaerr?' do |data|
+      expected, code = data
+      assert_equal(expected, Fluent::Win32Error.new(code).wsaerr?)
+    end
+  end
+
+  sub_test_case 'WindowsFile exceptions' do
+    test 'nothing raised' do
+      begin
+        path = "#{TMP_DIR}/test_windows_file.txt"
+        file1 = file2 = nil
+        file1 = File.open(path, "wb") do |f|
+        end
+        assert_nothing_raised do
+          file2 = Fluent::WindowsFile.new(path)
+        ensure
+          file2.close
+        end
+      ensure
+        file1.close if file1
+      end
+    end
+
+    test 'Errno::ENOENT raised' do
+      path = "#{TMP_DIR}/nofile.txt"
+      file = nil
+      assert_raise(Errno::ENOENT) do
+        file = Fluent::WindowsFile.new(path)
+      ensure
+        file.close if file
+      end
+    end
+
+    test 'ERROR_SHARING_VIOLATION raised' do
+      begin
+        path = "#{TMP_DIR}/test_windows_file.txt"
+        file1 = file2 = nil
+        file1 = File.open(path, "wb")
+        win32err = Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, path)
+        assert_raise(Errno::EACCES.new(win32err.message)) do
+          file2 = Fluent::WindowsFile.new(path, 'r', FILE_SHARE_READ)
+        ensure
+          file2.close if file2
+        end
+      ensure
+        file1.close if file1
+      end
+    end
+  end
+end if Fluent.windows?

data/test/plugin/test_in_exec.rb

@@ -255,7 +255,7 @@ EOC
     assert{ d.events.length > 0 }
     d.events.each do |event|
       assert_equal 'test', event[0]
-      assert_match /LoadError/, event[2]['message']
+      assert_match(/LoadError/, event[2]['message'])
     end
   end
 end

data/test/plugin/test_in_http.rb

@@ -770,6 +770,15 @@ class HttpInputTest < Test::Unit::TestCase
     end
   end
 
+  def test_get_request
+    d = create_driver(CONFIG)
+
+    d.run do
+      res = get("/cors.test", {}, {})
+      assert_equal "200", res.code
+    end
+  end
+
   def test_cors_preflight
     d = create_driver(CONFIG + 'cors_allow_origins ["*"]')
 
@@ -985,6 +994,12 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal $test_in_http_connection_object_ids[0], $test_in_http_connection_object_ids[1]
   end
 
+  def get(path, params, header = {})
+    http = Net::HTTP.new("127.0.0.1", PORT)
+    req = Net::HTTP::Get.new(path, header)
+    http.request(req)
+  end
+
   def options(path, params, header = {})
     http = Net::HTTP.new("127.0.0.1", PORT)
     req = Net::HTTP::Options.new(path, header)

data/test/plugin/test_in_tail.rb

@@ -6,6 +6,8 @@ require 'fluent/system_config'
 require 'net/http'
 require 'flexmock/test_unit'
 require 'timecop'
+require 'tmpdir'
+require 'securerandom'
 
 class TailInputTest < Test::Unit::TestCase
   include FlexMock::TestCase
@@ -22,28 +24,64 @@ class TailInputTest < Test::Unit::TestCase
   end
 
   def cleanup_directory(path)
-    begin
-      FileUtils.rm_f(path, secure: true)
-    rescue ArgumentError
-      FileUtils.rm_f(path) # For Ruby 2.6 or before.
+    unless Dir.exist?(path)
+      FileUtils.mkdir_p(path)
+      return
     end
-    if File.exist?(path)
-      FileUtils.remove_entry_secure(path, true)
+
+    if Fluent.windows?
+      Dir.glob("*", base: path).each do |name|
+        begin
+          cleanup_file(File.join(path, name))
+        rescue
+          # expect test driver block release already owned file handle.
+        end
+      end
+    else
+      begin
+        FileUtils.rm_f(path, secure:true)
+      rescue ArgumentError
+        FileUtils.rm_f(path) # For Ruby 2.6 or before.
+      end
+      if File.exist?(path)
+        FileUtils.remove_entry_secure(path, true)
+      end
     end
     FileUtils.mkdir_p(path)
   end
 
   def cleanup_file(path)
-    begin
-      FileUtils.rm_f(path, secure: true)
-    rescue ArgumentError
-      FileUtils.rm_f(path) # For Ruby 2.6 or before.
-    end
-    if File.exist?(path)
-      # ensure files are closed for Windows, on which deleted files
-      # are still visible from filesystem
-      GC.start(full_mark: true, immediate_mark: true, immediate_sweep: true)
-      FileUtils.remove_entry_secure(path, true)
+    if Fluent.windows?
+      # On Windows, when the file or directory is removed and created
+      # frequently, there is a case that creating file or directory will
+      # fail. This situation is caused by pending file or directory
+      # deletion which is mentioned on win32 API document [1]
+      # As a workaround, execute rename and remove method.
+      #
+      # [1] https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilea#files
+      #
+      file = File.join(Dir.tmpdir, SecureRandom.hex(10))
+      begin
+        FileUtils.mv(path, file)
+        FileUtils.rm_rf(file, secure: true)
+      rescue ArgumentError
+        FileUtils.rm_rf(file) # For Ruby 2.6 or before.
+      end
+      if File.exist?(file)
+        # ensure files are closed for Windows, on which deleted files
+        # are still visible from filesystem
+        GC.start(full_mark: true, immediate_mark: true, immediate_sweep: true)
+        FileUtils.remove_entry_secure(file, true)
+      end
+    else
+      begin
+        FileUtils.rm_f(path, secure: true)
+      rescue ArgumentError
+        FileUtils.rm_f(path) # For Ruby 2.6 or before.
+      end
+      if File.exist?(path)
+        FileUtils.remove_entry_secure(path, true)
+      end
     end
   end
 
@@ -118,6 +156,7 @@ class TailInputTest < Test::Unit::TestCase
       assert_equal 2, d.instance.rotate_wait
       assert_equal "#{TMP_DIR}/tail.pos", d.instance.pos_file
       assert_equal 1000, d.instance.read_lines_limit
+      assert_equal -1, d.instance.read_bytes_limit_per_second
       assert_equal false, d.instance.ignore_repeated_permission_error
       assert_nothing_raised do
         d.instance.have_read_capability?
@@ -157,6 +196,22 @@ class TailInputTest < Test::Unit::TestCase
       end
     end
 
+    sub_test_case "log throttling per file" do
+      test "w/o watcher timer is invalid" do
+        conf = CONFIG_ENABLE_WATCH_TIMER + config_element("ROOT", "", {"read_bytes_limit_per_second" => "8k"})
+        assert_raise(Fluent::ConfigError) do
+          create_driver(conf)
+        end
+      end
+
+      test "valid" do
+        conf = config_element("ROOT", "", {"read_bytes_limit_per_second" => "8k"})
+        assert_raise(Fluent::ConfigError) do
+          create_driver(conf)
+        end
+      end
+    end
+
     test "both enable_watch_timer and enable_stat_watcher are false" do
       assert_raise(Fluent::ConfigError) do
         create_driver(CONFIG_ENABLE_WATCH_TIMER + CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)
@@ -285,6 +340,138 @@ class TailInputTest < Test::Unit::TestCase
       assert num_events <= d.emit_count
     end
 
+    sub_test_case "log throttling per file" do
+      teardown do
+        cleanup_file("#{TMP_DIR}/tail.txt")
+      end
+
+      sub_test_case "reads_bytes_per_second w/o throttled" do
+        data("flat 8192 bytes, 2 events"        => [:flat, 100, 8192, 2, false],
+             "flat 8192 bytes, 2 events already read limit reached" => [:flat, 100, 8192, 2, true],
+             "flat 8192 bytes, 2 events w/o stat watcher" => [:flat_without_stat, 100, 8192, 2, false],
+             "flat #{8192*10} bytes, 20 events"  => [:flat, 100, (8192 * 10), 20],
+             "flat #{8192*10} bytes, 20 events w/o stat watcher"  => [:flat_without_stat, 100, (8192 * 10), 20],
+             "parse #{8192*4} bytes, 8 events"  => [:parse, 100, (8192 * 4), 8],
+             "parse #{8192*4} bytes, 8 events w/o stat watcher"  => [:parse_without_stat, 100, (8192 * 4), 8],
+             "parse #{8192*10} bytes, 20 events" => [:parse, 100, (8192 * 10), 20],
+             "parse #{8192*10} bytes, 20 events w/o stat watcher" => [:parse_without_stat, 100, (8192 * 10), 20],
+             "flat 8k bytes with unit, 2 events"        => [:flat, 100, "8k", 2],
+             "flat 8k bytes with unit, 2 events w/o stat watcher"        => [:flat_without_stat, 100, "8k", 2],
+             "flat #{8*10}k bytes with unit, 20 events"  => [:flat, 100, "#{8*10}k", 20],
+             "flat #{8*10}k bytes with unit, 20 events w/o stat watcher"  => [:flat_without_stat, 100, "#{8*10}k", 20],
+             "parse #{8*4}k bytes with unit, 8 events"  => [:parse, 100, "#{8*4}k", 8],
+             "parse #{8*4}k bytes with unit, 8 events w/o stat watcher"  => [:parse_without_stat, 100, "#{8*4}k", 8],
+             "parse #{8*10}k bytes with unit, 20 events" => [:parse, 100, "#{8*10}k", 20],
+             "parse #{8*10}k bytes with unit, 20 events w/o stat watcher" => [:parse_without_stat, 100, "#{8*10}k", 20])
+        def test_emit_with_read_bytes_limit_per_second(data)
+          config_style, limit, limit_bytes, num_events = data
+          case config_style
+          when :flat
+            config = CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes })
+          when :parse
+            config = CONFIG_READ_FROM_HEAD + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes }) + PARSE_SINGLE_LINE_CONFIG
+          when :flat_without_stat
+            config = CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + CONFIG_DISABLE_STAT_WATCHER + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes })
+          when :parse_without_stat
+            config = CONFIG_READ_FROM_HEAD + CONFIG_DISABLE_STAT_WATCHER + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes }) + PARSE_SINGLE_LINE_CONFIG
+          end
+
+          msg = 'test' * 2000 # in_tail reads 8192 bytes at once.
+          start_time = Fluent::Clock.now
+
+          d = create_driver(config)
+          d.run(expect_emits: 2) do
+            File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
+              100.times do
+                f.puts msg
+              end
+            }
+          end
+
+          assert_true(Fluent::Clock.now - start_time > 1)
+          assert_equal(num_events.times.map { {"message" => msg} },
+                       d.events.collect { |event| event[2] })
+        end
+
+        def test_read_bytes_limit_precede_read_lines_limit
+          config = CONFIG_READ_FROM_HEAD +
+                   SINGLE_LINE_CONFIG +
+                   config_element("", "", {
+                                    "read_lines_limit" => 1000,
+                                    "read_bytes_limit_per_second" => 8192
+                                  })
+          msg = 'abc'
+          start_time = Fluent::Clock.now
+          d = create_driver(config)
+          d.run(expect_emits: 2) do
+            File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
+              8000.times do
+                f.puts msg
+              end
+            }
+          end
+
+          assert_true(Fluent::Clock.now - start_time > 1)
+          assert_equal(4096.times.map { {"message" => msg} },
+                       d.events.collect { |event| event[2] })
+        end
+      end
+
+      sub_test_case "reads_bytes_per_second w/ throttled already" do
+        data("flat 8192 bytes, 2 events"        => [:flat, 100, 8192, 2],
+             "flat #{8192*10} bytes, 20 events"  => [:flat, 100, (8192 * 10), 20],
+             "parse #{8192*4} bytes, 8 events"  => [:parse, 100, (8192 * 4), 8],
+             "parse #{8192*10} bytes, 20 events" => [:parse, 100, (8192 * 10), 20],
+             "flat 8k bytes with unit, 2 events"        => [:flat, 100, "8k", 2],
+             "flat #{8*10}k bytes with unit, 20 events"  => [:flat, 100, "#{8*10}k", 20],
+             "parse #{8*4}k bytes with unit, 8 events"  => [:parse, 100, "#{8*4}k", 8],
+             "parse #{8*10}k bytes with unit, 20 events" => [:parse, 100, "#{8*10}k", 20])
+        def test_emit_with_read_bytes_limit_per_second(data)
+          config_style, limit, limit_bytes, num_events = data
+          case config_style
+          when :flat
+            config = CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes })
+          when :parse
+            config = CONFIG_READ_FROM_HEAD + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes }) + PARSE_SINGLE_LINE_CONFIG
+          end
+          d = create_driver(config)
+          msg = 'test' * 2000 # in_tail reads 8192 bytes at once.
+
+          mock.proxy(d.instance).io_handler(anything, anything) do |io_handler|
+            require 'fluent/config/types'
+            limit_bytes_value = Fluent::Config.size_value(limit_bytes)
+            io_handler.instance_variable_set(:@number_bytes_read, limit_bytes_value)
+            if Fluent.linux?
+              mock.proxy(io_handler).handle_notify.at_least(5)
+            else
+              mock.proxy(io_handler).handle_notify.twice
+            end
+            io_handler
+          end
+
+          File.open("#{TMP_DIR}/tail.txt", "ab") do |f|
+            100.times do
+              f.puts msg
+            end
+          end
+
+          # We should not do shutdown here due to hard timeout.
+          d.run do
+            start_time = Fluent::Clock.now
+            while Fluent::Clock.now - start_time < 0.8 do
+              File.open("#{TMP_DIR}/tail.txt", "ab") do |f|
+                f.puts msg
+                f.flush
+              end
+              sleep 0.05
+            end
+          end
+
+          assert_equal([], d.events)
+        end
+      end
+    end
+
     data(flat: CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG,
          parse: CONFIG_READ_FROM_HEAD + PARSE_SINGLE_LINE_CONFIG)
     def test_emit_with_read_from_head(data)
@@ -551,6 +738,7 @@ class TailInputTest < Test::Unit::TestCase
     File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
       f.puts "test1"
       f.puts "test2"
+      f.flush
     }
 
     d = create_driver(config)
@@ -560,19 +748,23 @@ class TailInputTest < Test::Unit::TestCase
         f.puts "test3\ntest4"
         f.flush
       }
-      sleep 1
+      waiting(2) { sleep 0.1 until d.events.length == 2 }
       File.truncate("#{TMP_DIR}/tail.txt", 6)
     end
 
-    events = d.events
-    assert_equal(3, events.length)
-    assert_equal({"message" => "test3"}, events[0][2])
-    assert_equal({"message" => "test4"}, events[1][2])
-    assert_equal({"message" => "test1"}, events[2][2])
-    assert(events[0][1].is_a?(Fluent::EventTime))
-    assert(events[1][1].is_a?(Fluent::EventTime))
-    assert(events[2][1].is_a?(Fluent::EventTime))
-    assert_equal(2, d.emit_count)
+    expected = {
+      emit_count: 2,
+      events: [
+        [Fluent::EventTime, {"message" => "test3"}],
+        [Fluent::EventTime, {"message" => "test4"}],
+        [Fluent::EventTime, {"message" => "test1"}],
+      ]
+    }
+    actual = {
+      emit_count: d.emit_count,
+      events: d.events.collect{|event| [event[1].class, event[2]]}
+    }
+    assert_equal(expected, actual)
   end
 
   def test_move_truncate_move_back
@@ -1876,4 +2068,95 @@ class TailInputTest < Test::Unit::TestCase
     waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size == 1 }
     d.instance_shutdown
   end
+
+  def test_ENOENT_error_after_setup_watcher
+    path = "#{TMP_DIR}/tail.txt"
+    FileUtils.touch(path)
+    config = config_element('', '', {
+                              'format' => 'none',
+                            })
+    d = create_driver(config)
+    mock.proxy(d.instance).setup_watcher(anything, anything) do |tw|
+      cleanup_file(path)
+      tw
+    end
+    assert_nothing_raised do
+      d.run(shutdown: false) {}
+    end
+    d.instance_shutdown
+    assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed with Errno::ENOENT. Drop tail watcher for now.\n") })
+  end
+
+  def test_EACCES_error_after_setup_watcher
+    path = "#{TMP_DIR}/noaccess/tail.txt"
+    begin
+      FileUtils.mkdir_p("#{TMP_DIR}/noaccess")
+      FileUtils.chmod(0755, "#{TMP_DIR}/noaccess")
+      FileUtils.touch(path)
+      config = config_element('', '', {
+                                'tag' => "tail",
+                                'path' => path,
+                                'format' => 'none',
+                              })
+      d = create_driver(config, false)
+      mock.proxy(d.instance).setup_watcher(anything, anything) do |tw|
+        FileUtils.chmod(0000, "#{TMP_DIR}/noaccess")
+        tw
+      end
+      assert_nothing_raised do
+        d.run(shutdown: false) {}
+      end
+      d.instance_shutdown
+      assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed with Errno::EACCES. Drop tail watcher for now.\n") })
+    end
+  ensure
+    FileUtils.chmod(0755, "#{TMP_DIR}/noaccess")
+    FileUtils.rm_rf("#{TMP_DIR}/noaccess")
+  end unless Fluent.windows?
+
+  def test_EACCES
+    path = "#{TMP_DIR}/tail.txt"
+    FileUtils.touch(path)
+    config = config_element('', '', {
+                              'format' => 'none',
+                            })
+    d = create_driver(config)
+    mock.proxy(Fluent::FileWrapper).stat(path) do |stat|
+      raise Errno::EACCES
+    end.at_least(1)
+    assert_nothing_raised do
+      d.run(shutdown: false) {}
+    end
+    d.instance_shutdown
+    assert($log.out.logs.any?{|log| log.include?("expand_paths: stat() for #{path} failed with Errno::EACCES. Skip file.\n") })
+  end
+
+  def test_shutdown_timeout
+    path = "#{TMP_DIR}/tail.txt"
+    File.open("#{TMP_DIR}/tail.txt", "wb") do |f|
+      (1024 * 1024 * 5).times do
+        f.puts "{\"test\":\"fizzbuzz\"}"
+      end
+    end
+
+    config =
+      CONFIG_READ_FROM_HEAD +
+      config_element('', '', {
+                              'format' => 'json',
+                              'skip_refresh_on_startup' => true,
+                            })
+    d = create_driver(config)
+    mock.proxy(d.instance).io_handler(anything, anything) do |io_handler|
+      io_handler.shutdown_timeout = 0.5
+      io_handler
+    end
+
+    start_time = Fluent::Clock.now
+    assert_nothing_raised do
+      d.run(expect_emits: 1)
+    end
+
+    elapsed = Fluent::Clock.now - start_time
+    assert_true(elapsed > 0.5 && elapsed < 2.5)
+  end
 end