fluentd 1.11.5-x86-mingw32 → 1.12.4-x86-mingw32

data/test/config/test_configurable.rb

@@ -1453,7 +1453,7 @@ module Fluent::Config
         @example = ConfigurableSpec::ExampleWithSkipAccessor.new
         @example.configure(config_element('ROOT'))
         assert_equal 'example7', @example.instance_variable_get(:@name)
-        assert_raise NoMethodError.new("undefined method `name' for #{@example}") do
+        assert_raise NoMethodError do
           @example.name
         end
       end

data/test/plugin/in_tail/test_position_file.rb

@@ -1,5 +1,6 @@
 require_relative '../../helper'
 require 'fluent/plugin/in_tail/position_file'
+require 'fluent/plugin/in_tail'
 
 require 'fileutils'
 require 'tempfile'
@@ -27,9 +28,15 @@ class IntailPositionFileTest < Test::Unit::TestCase
     f.seek(0)
   end
 
+  def follow_inodes_block
+    [true, false].each do |follow_inodes|
+      yield follow_inodes
+    end
+  end
+
   test '.load' do
     write_data(@file, TEST_CONTENT)
-    Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
+    Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
 
     @file.seek(0)
     lines = @file.readlines
@@ -41,7 +48,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
   sub_test_case '#try_compact' do
     test 'compact invalid and convert 32 bit inode value' do
       write_data(@file, TEST_CONTENT)
-      Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log).try_compact
+      Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log}).try_compact
 
       @file.seek(0)
       lines = @file.readlines
@@ -55,7 +62,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
         valid_path\t0000000000000002\t0000000000000001
         valid_path\t0000000000000003\t0000000000000004
       EOF
-      Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log).try_compact
+      Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log}).try_compact
 
       @file.seek(0)
       lines = @file.readlines
@@ -64,7 +71,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
 
     test 'does not change when the file is changed' do
       write_data(@file, TEST_CONTENT)
-      pf = Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log)
+      pf = Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log})
 
       mock.proxy(pf).fetch_compacted_entries do |r|
         @file.write("unwatched\t#{UNWATCHED_STR}\t0000000000000000\n")
@@ -78,12 +85,17 @@ class IntailPositionFileTest < Test::Unit::TestCase
       assert_equal 5, lines.size
     end
 
-    test 'update seek postion of remained position entry' do
-      pf = Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log)
-      pf['path1']
-      pf['path2']
-      pf['path3']
-      pf.unwatch('path1')
+    test 'update seek position of remained position entry' do
+      pf = Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log})
+      target_info1 = Fluent::Plugin::TailInput::TargetInfo.new('path1', -1)
+      target_info2 = Fluent::Plugin::TailInput::TargetInfo.new('path2', -1)
+      target_info3 = Fluent::Plugin::TailInput::TargetInfo.new('path3', -1)
+      pf[target_info1]
+      pf[target_info2]
+      pf[target_info3]
+
+      target_info1_2 = Fluent::Plugin::TailInput::TargetInfo.new('path1', 1234)
+      pf.unwatch(target_info1_2)
 
       pf.try_compact
 
@@ -93,8 +105,10 @@ class IntailPositionFileTest < Test::Unit::TestCase
       assert_equal "path3\t0000000000000000\t0000000000000000\n", lines[1]
       assert_equal 2, lines.size
 
-      pf.unwatch('path2')
-      pf.unwatch('path3')
+      target_info2_2 = Fluent::Plugin::TailInput::TargetInfo.new('path2', 1235)
+      target_info3_2 = Fluent::Plugin::TailInput::TargetInfo.new('path3', 1236)
+      pf.unwatch(target_info2_2)
+      pf.unwatch(target_info3_2)
       @file.seek(0)
       lines = @file.readlines
       assert_equal "path2\t#{UNWATCHED_STR}\t0000000000000000\n", lines[0]
@@ -106,7 +120,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
   sub_test_case '#load' do
     test 'compact invalid and convert 32 bit inode value' do
       write_data(@file, TEST_CONTENT)
-      Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
+      Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
 
       @file.seek(0)
       lines = @file.readlines
@@ -120,7 +134,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
         valid_path\t0000000000000002\t0000000000000001
         valid_path\t0000000000000003\t0000000000000004
       EOF
-      Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log).load
+      Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log}).load
 
       @file.seek(0)
       lines = @file.readlines
@@ -131,9 +145,10 @@ class IntailPositionFileTest < Test::Unit::TestCase
   sub_test_case '#[]' do
     test 'return entry' do
       write_data(@file, TEST_CONTENT)
-      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
+      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
 
-      f = pf['valid_path']
+      valid_target_info = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', File.stat(@file).ino)
+      f = pf[valid_target_info]
       assert_equal Fluent::Plugin::TailInput::FilePositionEntry, f.class
       assert_equal 2, f.read_pos
       assert_equal 1, f.read_inode
@@ -142,7 +157,8 @@ class IntailPositionFileTest < Test::Unit::TestCase
       lines = @file.readlines
       assert_equal 2, lines.size
 
-      f = pf['nonexist_path']
+      nonexistent_target_info = Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)
+      f = pf[nonexistent_target_info]
       assert_equal Fluent::Plugin::TailInput::FilePositionEntry, f.class
       assert_equal 0, f.read_pos
       assert_equal 0, f.read_inode
@@ -155,19 +171,19 @@ class IntailPositionFileTest < Test::Unit::TestCase
 
     test 'does not change other value position if other entry try to write' do
       write_data(@file, TEST_CONTENT)
-      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
+      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, logger: $log)
 
-      f = pf['nonexist_path']
+      f = pf[Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)]
       assert_equal 0, f.read_inode
       assert_equal 0, f.read_pos
 
-      pf['valid_path'].update(1, 2)
+      pf[Fluent::Plugin::TailInput::TargetInfo.new('valid_path', File.stat(@file).ino)].update(1, 2)
 
-      f = pf['nonexist_path']
+      f = pf[Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)]
       assert_equal 0, f.read_inode
       assert_equal 0, f.read_pos
 
-      pf['nonexist_path'].update(1, 2)
+      pf[Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)].update(1, 2)
       assert_equal 1, f.read_inode
       assert_equal 2, f.read_pos
     end
@@ -176,14 +192,18 @@ class IntailPositionFileTest < Test::Unit::TestCase
   sub_test_case '#unwatch' do
     test 'deletes entry by path' do
       write_data(@file, TEST_CONTENT)
-      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
-      p1 = pf['valid_path']
+      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, logger: $log)
+      inode1 = File.stat(@file).ino
+      target_info1 = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', inode1)
+      p1 = pf[target_info1]
       assert_equal Fluent::Plugin::TailInput::FilePositionEntry, p1.class
 
-      pf.unwatch('valid_path')
+      pf.unwatch(target_info1)
       assert_equal p1.read_pos, Fluent::Plugin::TailInput::PositionFile::UNWATCHED_POSITION
 
-      p2 = pf['valid_path']
+      inode2 = File.stat(@file).ino
+      target_info2 = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', inode2)
+      p2 = pf[target_info2]
       assert_equal Fluent::Plugin::TailInput::FilePositionEntry, p2.class
 
       assert_not_equal p1, p2
@@ -262,4 +282,58 @@ class IntailPositionFileTest < Test::Unit::TestCase
       assert_equal 2, f.read_inode
     end
   end
+
+  sub_test_case "TargetInfo equality rules" do
+    sub_test_case "== operator" do
+      def test_equal
+        t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
+        t2 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1235)
+
+        assert_equal t1, t2
+      end
+
+      def test_not_equal
+        t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
+        t2 = Fluent::Plugin::TailInput::TargetInfo.new("test2", 1234)
+
+        assert_not_equal t1, t2
+      end
+    end
+
+    sub_test_case "eql? method" do
+      def test_eql?
+        t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
+        t2 = Fluent::Plugin::TailInput::TargetInfo.new("test", 5321)
+
+        assert do
+          t1.eql? t2
+        end
+      end
+
+      def test_not_eql?
+        t1 = Fluent::Plugin::TailInput::TargetInfo.new("test2", 1234)
+        t2 = Fluent::Plugin::TailInput::TargetInfo.new("test3", 1234)
+
+        assert do
+          !t1.eql? t2
+        end
+      end
+    end
+
+    sub_test_case "hash" do
+      def test_equal
+        t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
+        t2 = Fluent::Plugin::TailInput::TargetInfo.new("test", 7321)
+
+        assert_equal t1.hash, t2.hash
+      end
+
+      def test_not_equal
+        t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
+        t2 = Fluent::Plugin::TailInput::TargetInfo.new("test2", 1234)
+
+        assert_not_equal t1.hash, t2.hash
+      end
+    end
+  end
 end

data/test/plugin/test_file_wrapper.rb

@@ -0,0 +1,105 @@
+require_relative '../helper'
+require 'fluent/plugin/file_wrapper'
+
+class FileWrapperTest < Test::Unit::TestCase
+  require 'windows/file'
+  require 'windows/error'
+  include Windows::File
+  include Windows::Error
+
+  TMP_DIR = File.dirname(__FILE__) + "/../tmp/file_wrapper#{ENV['TEST_ENV_NUMBER']}"
+
+  def setup
+    FileUtils.mkdir_p(TMP_DIR)
+  end
+
+  def teardown
+    FileUtils.rm_rf(TMP_DIR)
+  end
+
+  sub_test_case 'Win32Error' do
+    test 'equal' do
+      assert_equal(Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "message"),
+                   Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "message"))
+    end
+
+    test 'different error code' do
+      assert_not_equal(Fluent::Win32Error.new(ERROR_FILE_NOT_FOUND),
+                       Fluent::Win32Error.new(ERROR_SHARING_VIOLATION))
+    end
+
+    test 'different error message' do
+      assert_not_equal(Fluent::Win32Error.new(ERROR_FILE_NOT_FOUND, "message1"),
+                       Fluent::Win32Error.new(ERROR_FILE_NOT_FOUND, "message2"))
+    end
+
+    test 'different class' do
+      assert_not_equal(Errno::EPIPE,
+                       Fluent::Win32Error.new(ERROR_SHARING_VIOLATION))
+    end
+
+    test 'ERROR_SHARING_VIOLATION message' do
+      assert_equal(Fluent::Win32Error.new(ERROR_SHARING_VIOLATION).message,
+                   "Fluent::Win32Error: code: 32, The process cannot access the file because it is being used by another process.")
+    end
+
+    test 'ERROR_SHARING_VIOLATION with a message' do
+      assert_equal(Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "cannot open the file").message,
+                   "Fluent::Win32Error: code: 32, The process cannot access the file because it is being used by another process." +
+                   " - cannot open the file")
+    end
+
+    test 'to_s' do
+      assert_equal("Fluent::Win32Error: code: 32, The process cannot access the file because it is being used by another process. - C:\file.txt",
+                   Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "C:\file.txt").to_s)
+    end
+
+    test 'inspect' do
+      assert_equal("#<Fluent::Win32Error: code: 32, The process cannot access the file because it is being used by another process. - C:\file.txt>",
+                   Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "C:\file.txt").inspect)
+    end
+  end
+
+  sub_test_case 'WindowsFile exceptions' do
+    test 'nothing raised' do
+      begin
+        path = "#{TMP_DIR}/test_windows_file.txt"
+        file1 = file2 = nil
+        file1 = File.open(path, "wb") do |f|
+        end
+        assert_nothing_raised do
+          file2 = Fluent::WindowsFile.new(path)
+        ensure
+          file2.close
+        end
+      ensure
+        file1.close if file1
+      end
+    end
+
+    test 'Errno::ENOENT raised' do
+      path = "#{TMP_DIR}/nofile.txt"
+      file = nil
+      assert_raise(Errno::ENOENT) do
+        file = Fluent::WindowsFile.new(path)
+      ensure
+        file.close if file
+      end
+    end
+
+    test 'ERROR_SHARING_VIOLATION raised' do
+      begin
+        path = "#{TMP_DIR}/test_windows_file.txt"
+        file1 = file2 = nil
+        file1 = File.open(path, "wb")
+        assert_raise(Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, path)) do
+          file2 = Fluent::WindowsFile.new(path, 'r', FILE_SHARE_READ)
+        ensure
+          file2.close if file2
+        end
+      ensure
+        file1.close if file1
+      end
+    end
+  end
+end if Fluent.windows?

data/test/plugin/test_in_exec.rb

@@ -255,7 +255,7 @@ EOC
     assert{ d.events.length > 0 }
     d.events.each do |event|
       assert_equal 'test', event[0]
-      assert_match /LoadError/, event[2]['message']
+      assert_match(/LoadError/, event[2]['message'])
     end
   end
 end

data/test/plugin/test_in_http.rb

@@ -46,6 +46,7 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal 10*1024*1024, d.instance.body_size_limit
     assert_equal 5, d.instance.keepalive_timeout
     assert_equal false, d.instance.add_http_headers
+    assert_equal false, d.instance.add_query_params
   end
 
   def test_time
@@ -907,6 +908,30 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal ["403", "403"], res_codes
   end
 
+  def test_add_query_params 
+    d = create_driver(CONFIG + "add_query_params true")
+    assert_equal true, d.instance.add_query_params
+
+    time = event_time("2011-01-02 13:14:15 UTC")
+    time_i = time.to_i
+    events = [
+      ["tag1", time, {"a"=>1, "QUERY_A"=>"b"}],
+      ["tag2", time, {"a"=>2, "QUERY_A"=>"b"}],
+    ]
+    res_codes = []
+    res_bodies = []
+
+    d.run do
+      events.each do |tag, _t, record|
+        res = post("/#{tag}?a=b", {"json"=>record.to_json, "time"=>time_i.to_s})
+        res_codes << res.code
+      end
+    end
+    assert_equal ["200", "200"], res_codes
+    assert_equal [], res_bodies
+    assert_equal events, d.events
+  end
+
   $test_in_http_connection_object_ids = []
   $test_in_http_content_types = []
   $test_in_http_content_types_flag = false

data/test/plugin/test_in_tail.rb

@@ -6,6 +6,8 @@ require 'fluent/system_config'
 require 'net/http'
 require 'flexmock/test_unit'
 require 'timecop'
+require 'tmpdir'
+require 'securerandom'
 
 class TailInputTest < Test::Unit::TestCase
   include FlexMock::TestCase
@@ -17,29 +19,98 @@ class TailInputTest < Test::Unit::TestCase
 
   def teardown
     super
+    cleanup_directory(TMP_DIR)
     Fluent::Engine.stop
   end
 
   def cleanup_directory(path)
-    FileUtils.rm_rf(path, secure: true)
-    if File.exist?(path)
-      FileUtils.remove_entry_secure(path, true)
+    unless Dir.exist?(path)
+      FileUtils.mkdir_p(path)
+      return
+    end
+
+    if Fluent.windows?
+      Dir.glob("*", base: path).each do |name|
+        begin
+          cleanup_file(File.join(path, name))
+        rescue
+          # expect test driver block release already owned file handle.
+        end
+      end
+    else
+      begin
+        FileUtils.rm_f(path, secure:true)
+      rescue ArgumentError
+        FileUtils.rm_f(path) # For Ruby 2.6 or before.
+      end
+      if File.exist?(path)
+        FileUtils.remove_entry_secure(path, true)
+      end
     end
     FileUtils.mkdir_p(path)
   end
 
+  def cleanup_file(path)
+    if Fluent.windows?
+      # On Windows, when the file or directory is removed and created
+      # frequently, there is a case that creating file or directory will
+      # fail. This situation is caused by pending file or directory
+      # deletion which is mentioned on win32 API document [1]
+      # As a workaround, execute rename and remove method.
+      #
+      # [1] https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilea#files
+      #
+      file = File.join(Dir.tmpdir, SecureRandom.hex(10))
+      begin
+        FileUtils.mv(path, file)
+        FileUtils.rm_rf(file, secure: true)
+      rescue ArgumentError
+        FileUtils.rm_rf(file) # For Ruby 2.6 or before.
+      end
+      if File.exist?(file)
+        # ensure files are closed for Windows, on which deleted files
+        # are still visible from filesystem
+        GC.start(full_mark: true, immediate_mark: true, immediate_sweep: true)
+        FileUtils.remove_entry_secure(file, true)
+      end
+    else
+      begin
+        FileUtils.rm_f(path, secure: true)
+      rescue ArgumentError
+        FileUtils.rm_f(path) # For Ruby 2.6 or before.
+      end
+      if File.exist?(path)
+        FileUtils.remove_entry_secure(path, true)
+      end
+    end
+  end
+
+  def create_target_info(path)
+    Fluent::Plugin::TailInput::TargetInfo.new(path, Fluent::FileWrapper.stat(path).ino)
+  end
+
   TMP_DIR = File.dirname(__FILE__) + "/../tmp/tail#{ENV['TEST_ENV_NUMBER']}"
 
   CONFIG = config_element("ROOT", "", {
                             "path" => "#{TMP_DIR}/tail.txt",
                             "tag" => "t1",
-                            "rotate_wait" => "2s"
+                            "rotate_wait" => "2s",
+                            "refresh_interval" => "1s"
                           })
   COMMON_CONFIG = CONFIG + config_element("", "", { "pos_file" => "#{TMP_DIR}/tail.pos" })
   CONFIG_READ_FROM_HEAD = config_element("", "", { "read_from_head" => true })
   CONFIG_ENABLE_WATCH_TIMER = config_element("", "", { "enable_watch_timer" => false })
   CONFIG_DISABLE_STAT_WATCHER = config_element("", "", { "enable_stat_watcher" => false })
   CONFIG_OPEN_ON_EVERY_UPDATE = config_element("", "", { "open_on_every_update" => true })
+  COMMON_FOLLOW_INODE_CONFIG = config_element("ROOT", "", {
+                                                "path" => "#{TMP_DIR}/tail.txt*",
+                                                "pos_file" => "#{TMP_DIR}/tail.pos",
+                                                "tag" => "t1",
+                                                "refresh_interval" => "1s",
+                                                "read_from_head" => "true",
+                                                "format" => "none",
+                                                "follow_inodes" => "true"
+                                              })
   SINGLE_LINE_CONFIG = config_element("", "", { "format" => "/(?<message>.*)/" })
   PARSE_SINGLE_LINE_CONFIG = config_element("", "", {}, [config_element("parse", "", { "@type" => "/(?<message>.*)/" })])
   MULTILINE_CONFIG = config_element(
@@ -86,6 +157,9 @@ class TailInputTest < Test::Unit::TestCase
       assert_equal "#{TMP_DIR}/tail.pos", d.instance.pos_file
       assert_equal 1000, d.instance.read_lines_limit
       assert_equal false, d.instance.ignore_repeated_permission_error
+      assert_nothing_raised do
+        d.instance.have_read_capability?
+      end
     end
 
     data("empty" => config_element,
@@ -115,6 +189,12 @@ class TailInputTest < Test::Unit::TestCase
       end
     end
 
+    test "follow_inodes w/o pos file" do
+      assert_raise(Fluent::ConfigError) do
+        create_driver(CONFIG + config_element('', '', {'follow_inodes' => 'true'}))
+      end
+    end
+
     test "both enable_watch_timer and enable_stat_watcher are false" do
       assert_raise(Fluent::ConfigError) do
         create_driver(CONFIG_ENABLE_WATCH_TIMER + CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)
@@ -229,7 +309,7 @@ class TailInputTest < Test::Unit::TestCase
       d = create_driver(config)
       msg = 'test' * 2000 # in_tail reads 8192 bytes at once.
 
-      d.run(expect_emits: num_events, timeout: 1) do
+      d.run(expect_emits: num_events, timeout: 2) do
         File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
           f.puts msg
           f.puts msg
@@ -509,6 +589,7 @@ class TailInputTest < Test::Unit::TestCase
     File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
       f.puts "test1"
       f.puts "test2"
+      f.flush
     }
 
     d = create_driver(config)
@@ -518,19 +599,23 @@ class TailInputTest < Test::Unit::TestCase
         f.puts "test3\ntest4"
         f.flush
       }
-      sleep 1
+      waiting(2) { sleep 0.1 until d.events.length == 2 }
       File.truncate("#{TMP_DIR}/tail.txt", 6)
     end
 
-    events = d.events
-    assert_equal(3, events.length)
-    assert_equal({"message" => "test3"}, events[0][2])
-    assert_equal({"message" => "test4"}, events[1][2])
-    assert_equal({"message" => "test1"}, events[2][2])
-    assert(events[0][1].is_a?(Fluent::EventTime))
-    assert(events[1][1].is_a?(Fluent::EventTime))
-    assert(events[2][1].is_a?(Fluent::EventTime))
-    assert_equal(2, d.emit_count)
+    expected = {
+      emit_count: 2,
+      events: [
+        [Fluent::EventTime, {"message" => "test3"}],
+        [Fluent::EventTime, {"message" => "test4"}],
+        [Fluent::EventTime, {"message" => "test1"}],
+      ]
+    }
+    actual = {
+      emit_count: d.emit_count,
+      events: d.events.collect{|event| [event[1].class, event[2]]}
+    }
+    assert_equal(expected, actual)
   end
 
   def test_move_truncate_move_back
@@ -999,6 +1084,7 @@ class TailInputTest < Test::Unit::TestCase
     # * path test
     # TODO: Clean up tests
     EX_ROTATE_WAIT = 0
+    EX_FOLLOW_INODES = false
 
     EX_CONFIG = config_element("", "", {
                                  "tag" => "tail",
@@ -1008,38 +1094,43 @@ class TailInputTest < Test::Unit::TestCase
                                  "read_from_head" => true,
                                  "refresh_interval" => 30,
                                  "rotate_wait" => "#{EX_ROTATE_WAIT}s",
+                                 "follow_inodes" => "#{EX_FOLLOW_INODES}",
                                })
-    EX_PATHS = [
-      'test/plugin/data/2010/01/20100102-030405.log',
-      'test/plugin/data/log/foo/bar.log',
-      'test/plugin/data/log/test.log'
-    ]
-
     def test_expand_paths
+      ex_paths = [
+        create_target_info('test/plugin/data/2010/01/20100102-030405.log'),
+        create_target_info('test/plugin/data/log/foo/bar.log'),
+        create_target_info('test/plugin/data/log/test.log')
+      ]
       plugin = create_driver(EX_CONFIG, false).instance
       flexstub(Time) do |timeclass|
         timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 2, 3, 4, 5))
-        assert_equal EX_PATHS, plugin.expand_paths.sort
+        assert_equal ex_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
       end
 
       # Test exclusion
-      exclude_config = EX_CONFIG + config_element("", "", { "exclude_path" => %Q(["#{EX_PATHS.last}"]) })
+      exclude_config = EX_CONFIG + config_element("", "", { "exclude_path" => %Q(["#{ex_paths.last.path}"]) })
       plugin = create_driver(exclude_config, false).instance
-      assert_equal EX_PATHS - [EX_PATHS.last], plugin.expand_paths.sort
+      assert_equal ex_paths - [ex_paths.last], plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
     end
 
     def test_expand_paths_with_duplicate_configuration
       expanded_paths = [
-        'test/plugin/data/log/foo/bar.log',
-        'test/plugin/data/log/test.log'
+        create_target_info('test/plugin/data/log/foo/bar.log'),
+        create_target_info('test/plugin/data/log/test.log')
       ]
       duplicate_config = EX_CONFIG.dup
       duplicate_config["path"]="test/plugin/data/log/**/*.log, test/plugin/data/log/**/*.log"
       plugin = create_driver(EX_CONFIG, false).instance
-      assert_equal expanded_paths, plugin.expand_paths.sort
+      assert_equal expanded_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
     end
 
     def test_expand_paths_with_timezone
+      ex_paths = [
+        create_target_info('test/plugin/data/2010/01/20100102-030405.log'),
+        create_target_info('test/plugin/data/log/foo/bar.log'),
+        create_target_info('test/plugin/data/log/test.log')
+      ]
       ['Asia/Taipei', '+08'].each do |tz_type|
         taipei_config = EX_CONFIG + config_element("", "", {"path_timezone" => tz_type})
         plugin = create_driver(taipei_config, false).instance
@@ -1053,8 +1144,8 @@ class TailInputTest < Test::Unit::TestCase
             # env : 2010-01-01 19:04:05 (UTC), tail path : 2010-01-02 03:04:05 (Asia/Taipei)
             timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 1, 19, 4, 5))
 
-            assert_equal EX_PATHS, plugin.expand_paths.sort
-            assert_equal EX_PATHS - [EX_PATHS.first], exclude_plugin.expand_paths.sort
+            assert_equal ex_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
+            assert_equal ex_paths - [ex_paths.first], exclude_plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
           end
         end
       end
@@ -1062,10 +1153,10 @@ class TailInputTest < Test::Unit::TestCase
 
     def test_log_file_without_extension
       expected_files = [
-        'test/plugin/data/log/bar',
-        'test/plugin/data/log/foo/bar.log',
-        'test/plugin/data/log/foo/bar2',
-        'test/plugin/data/log/test.log'
+        create_target_info('test/plugin/data/log/bar'),
+        create_target_info('test/plugin/data/log/foo/bar.log'),
+        create_target_info('test/plugin/data/log/foo/bar2'),
+        create_target_info('test/plugin/data/log/test.log')
       ]
 
       config = config_element("", "", {
@@ -1076,7 +1167,7 @@ class TailInputTest < Test::Unit::TestCase
       })
 
       plugin = create_driver(config, false).instance
-      assert_equal expected_files, plugin.expand_paths.sort
+      assert_equal expected_files, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
     end
 
     def test_unwatched_files_should_be_removed
@@ -1112,6 +1203,49 @@ class TailInputTest < Test::Unit::TestCase
     end
   end
 
+  sub_test_case "path w/ Linux capability" do
+    def capability_enabled?
+      if Fluent.linux?
+        begin
+          require 'capng'
+          true
+        rescue LoadError
+          false
+        end
+      else
+        false
+      end
+    end
+
+    setup do
+      omit "This environment is not enabled Linux capability handling feature" unless capability_enabled?
+
+      @capng = CapNG.new(:current_process)
+      flexstub(Fluent::Capability) do |klass|
+        klass.should_receive(:new).with(:current_process).and_return(@capng)
+      end
+    end
+
+    data("dac_read_search" => [:dac_read_search, true,  1],
+         "dac_override"    => [:dac_override,    true,  1],
+         "chown"           => [:chown,           false, 0],
+        )
+    test "with partially elevated privileges" do |data|
+      cap, result, readable_paths = data
+      @capng.update(:add, :effective, cap)
+
+      d = create_driver(
+        config_element("ROOT", "", {
+                            "path" => "/var/log/ker*.log", # Use /var/log/kern.log
+                            "tag" => "t1",
+                            "rotate_wait" => "2s"
+                       }) + PARSE_SINGLE_LINE_CONFIG, false)
+
+      assert_equal readable_paths, d.instance.expand_paths.length
+      assert_equal result, d.instance.have_read_capability?
+    end
+  end
+
   def test_pos_file_dir_creation
     config = config_element("", "", {
       "tag" => "tail",
@@ -1159,25 +1293,35 @@ class TailInputTest < Test::Unit::TestCase
   end
 
   def test_z_refresh_watchers
+    ex_paths = [
+      create_target_info('test/plugin/data/2010/01/20100102-030405.log'),
+      create_target_info('test/plugin/data/log/foo/bar.log'),
+      create_target_info('test/plugin/data/log/test.log'),
+    ]
     plugin = create_driver(EX_CONFIG, false).instance
     sio = StringIO.new
     plugin.instance_eval do
-      @pf = Fluent::Plugin::TailInput::PositionFile.load(sio, logger: $log)
+      @pf = Fluent::Plugin::TailInput::PositionFile.load(sio, EX_FOLLOW_INODES, {}, logger: $log)
       @loop = Coolio::Loop.new
     end
 
     Timecop.freeze(2010, 1, 2, 3, 4, 5) do
-      EX_PATHS.each do |path|
-        mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(path, anything, anything, true, anything, nil, anything).once
+      ex_paths.each do |target_info|
+        mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything).once
       end
 
       plugin.refresh_watchers
     end
 
-    mock.proxy(plugin).detach_watcher_after_rotate_wait(plugin.instance_variable_get(:@tails)['test/plugin/data/2010/01/20100102-030405.log'])
+    path = 'test/plugin/data/2010/01/20100102-030405.log'
+    target_info = Fluent::Plugin::TailInput::TargetInfo.new(path, Fluent::FileWrapper.stat(path).ino)
+    mock.proxy(plugin).detach_watcher_after_rotate_wait(plugin.instance_variable_get(:@tails)[target_info], target_info.ino)
 
     Timecop.freeze(2010, 1, 2, 3, 4, 6) do
-      mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new('test/plugin/data/2010/01/20100102-030406.log', anything, anything, true, anything, nil, anything).once
+      path = "test/plugin/data/2010/01/20100102-030406.log"
+      inode = Fluent::FileWrapper.stat(path).ino
+      target_info = Fluent::Plugin::TailInput::TargetInfo.new(path, inode)
+      mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything).once
       plugin.refresh_watchers
 
       flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
@@ -1323,6 +1467,305 @@ class TailInputTest < Test::Unit::TestCase
     end
   end
 
+  sub_test_case 'inode_processing' do
+    def test_should_delete_file_pos_entry_for_non_existing_file_with_follow_inodes
+      config = COMMON_FOLLOW_INODE_CONFIG
+
+      path = "#{TMP_DIR}/tail.txt"
+      ino = 1
+      pos = 1234
+      File.open("#{TMP_DIR}/tail.pos", "wb") {|f|
+        f.puts ("%s\t%016x\t%016x\n" % [path, pos, ino])
+      }
+
+      d = create_driver(config, false)
+      d.run
+
+      pos_file = File.open("#{TMP_DIR}/tail.pos", "r")
+      pos_file.pos = 0
+
+      assert_raise(EOFError) do
+        pos_file.readline
+      end
+    end
+
+    def test_should_write_latest_offset_after_rotate_wait
+      config = COMMON_FOLLOW_INODE_CONFIG
+      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+        f.puts "test1"
+        f.puts "test2"
+      }
+
+      d = create_driver(config, false)
+      d.run(expect_emits: 2, shutdown: false) do
+        File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
+        FileUtils.move("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail.txt" + "1")
+        sleep 1
+        File.open("#{TMP_DIR}/tail.txt" + "1", "ab") {|f| f.puts "test4\n"}
+      end
+
+      pos_file = File.open("#{TMP_DIR}/tail.pos", "r")
+      pos_file.pos = 0
+      line_parts = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
+      waiting(5) {
+        while line_parts[2].to_i(16) != 24
+          sleep(0.1)
+          pos_file.pos = 0
+          line_parts = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
+        end
+      }
+      assert_equal(24, line_parts[2].to_i(16))
+      d.instance_shutdown
+    end
+
+    def test_should_keep_and_update_existing_file_pos_entry_for_deleted_file_when_new_file_with_same_name_created
+      config = config_element("", "", {"format" => "none"})
+
+      path = "#{TMP_DIR}/tail.txt"
+      ino = 1
+      pos = 1234
+      File.open("#{TMP_DIR}/tail.pos", "wb") {|f|
+        f.puts ("%s\t%016x\t%016x\n" % [path, pos, ino])
+      }
+
+      d = create_driver(config)
+      d.run(shutdown: false)
+
+      pos_file = File.open("#{TMP_DIR}/tail.pos", "r")
+      pos_file.pos = 0
+
+      path_pos_ino = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
+      assert_equal(path, path_pos_ino[1])
+      assert_equal(pos, path_pos_ino[2].to_i(16))
+      assert_equal(ino, path_pos_ino[3].to_i(16))
+
+      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+        f.puts "test1"
+        f.puts "test2"
+      }
+      Timecop.travel(Time.now + 10) do
+        sleep 5
+        pos_file.pos = 0
+        tuple = create_target_info("#{TMP_DIR}/tail.txt")
+        path_pos_ino = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
+        assert_equal(tuple.path, path_pos_ino[1])
+        assert_equal(12, path_pos_ino[2].to_i(16))
+        assert_equal(tuple.ino, path_pos_ino[3].to_i(16))
+      end
+      d.instance_shutdown
+    end
+
+    def test_should_mark_file_unwatched_after_limit_recently_modified_and_rotate_wait
+      config = config_element("ROOT", "", {
+          "path" => "#{TMP_DIR}/tail.txt*",
+          "pos_file" => "#{TMP_DIR}/tail.pos",
+          "tag" => "t1",
+          "rotate_wait" => "1s",
+          "refresh_interval" => "1s",
+          "limit_recently_modified" => "1s",
+          "read_from_head" => "true",
+          "format" => "none",
+          "follow_inodes" => "true",
+      })
+
+      d = create_driver(config, false)
+
+      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+        f.puts "test1"
+        f.puts "test2"
+      }
+      target_info = create_target_info("#{TMP_DIR}/tail.txt")
+
+      d.run(expect_emits: 1, shutdown: false) do
+        File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
+      end
+
+
+      Timecop.travel(Time.now + 10) do
+        waiting(5) {
+          # @pos will be reset as 0 when UNWATCHED_POSITION is specified.
+          sleep 0.1 until d.instance.instance_variable_get(:@pf)[target_info].read_pos == 0
+        }
+      end
+
+      assert_equal(0, d.instance.instance_variable_get(:@pf)[target_info].read_pos)
+
+      d.instance_shutdown
+    end
+
+    def test_should_read_from_head_on_file_renaming_with_star_in_pattern
+      config = config_element("ROOT", "", {
+          "path" => "#{TMP_DIR}/tail.txt*",
+          "pos_file" => "#{TMP_DIR}/tail.pos",
+          "tag" => "t1",
+          "rotate_wait" => "10s",
+          "refresh_interval" => "1s",
+          "limit_recently_modified" => "60s",
+          "read_from_head" => "true",
+          "format" => "none",
+          "follow_inodes" => "true"
+      })
+
+      d = create_driver(config, false)
+
+      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+        f.puts "test1"
+        f.puts "test2"
+      }
+
+      d.run(expect_emits: 2, shutdown: false) do
+        File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
+        FileUtils.move("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail.txt1")
+      end
+
+      events = d.events
+      assert_equal(3, events.length)
+      d.instance_shutdown
+    end
+
+    def test_should_not_read_from_head_on_rotation_when_watching_inodes
+      config = COMMON_FOLLOW_INODE_CONFIG
+
+      d = create_driver(config, false)
+
+      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+        f.puts "test1"
+        f.puts "test2"
+      }
+
+      d.run(expect_emits: 1, shutdown: false) do
+        File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
+      end
+
+      FileUtils.move("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail.txt1")
+      Timecop.travel(Time.now + 10) do
+        sleep 2
+        events = d.events
+        assert_equal(3, events.length)
+      end
+
+      d.instance_shutdown
+    end
+
+    def test_should_mark_file_unwatched_if_same_name_file_created_with_different_inode
+      config = COMMON_FOLLOW_INODE_CONFIG
+
+      d = create_driver(config, false)
+
+      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+        f.puts "test1"
+        f.puts "test2"
+      }
+      target_info = create_target_info("#{TMP_DIR}/tail.txt")
+
+      d.run(expect_emits: 2, shutdown: false) do
+        File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
+        cleanup_file("#{TMP_DIR}/tail.txt")
+        File.open("#{TMP_DIR}/tail.txt", "wb") {|f| f.puts "test4\n"}
+      end
+
+      new_target_info = create_target_info("#{TMP_DIR}/tail.txt")
+
+      pos_file = d.instance.instance_variable_get(:@pf)
+
+      waiting(10) {
+        # @pos will be reset as 0 when UNWATCHED_POSITION is specified.
+        sleep 0.1 until pos_file[target_info].read_pos == 0
+      }
+      new_position = pos_file[new_target_info].read_pos
+      assert_equal(6, new_position)
+
+      d.instance_shutdown
+    end
+
+    def test_should_close_watcher_after_rotate_wait
+      now = Time.now
+      config = COMMON_FOLLOW_INODE_CONFIG + config_element('', '', {"rotate_wait" => "1s", "limit_recently_modified" => "1s"})
+
+      d = create_driver(config, false)
+
+      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+        f.puts "test1"
+        f.puts "test2"
+      }
+      target_info = create_target_info("#{TMP_DIR}/tail.txt")
+      mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, true, anything, nil, anything).once
+      d.run(shutdown: false)
+      assert d.instance.instance_variable_get(:@tails)[target_info]
+
+      Timecop.travel(now + 10) do
+        d.instance.instance_eval do
+          sleep 0.1 until @tails[target_info] == nil
+        end
+        assert_nil d.instance.instance_variable_get(:@tails)[target_info]
+      end
+      d.instance_shutdown
+    end
+
+    def test_should_create_new_watcher_for_new_file_with_same_name
+      now = Time.now
+      config = COMMON_FOLLOW_INODE_CONFIG + config_element('', '', {"limit_recently_modified" => "2s"})
+
+      d = create_driver(config, false)
+
+      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+        f.puts "test1"
+        f.puts "test2"
+      }
+      path_ino = create_target_info("#{TMP_DIR}/tail.txt")
+
+      d.run(expect_emits: 1, shutdown: false) do
+        File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
+      end
+
+      cleanup_file("#{TMP_DIR}/tail.txt")
+      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+        f.puts "test3"
+        f.puts "test4"
+      }
+      new_path_ino = create_target_info("#{TMP_DIR}/tail.txt")
+
+      Timecop.travel(now + 10) do
+        sleep 3
+        d.instance.instance_eval do
+          @tails[path_ino] == nil
+          @tails[new_path_ino] != nil
+        end
+      end
+
+      events = d.events
+
+      assert_equal(5, events.length)
+
+      d.instance_shutdown
+    end
+
+    def test_truncate_file_with_follow_inodes
+      config = COMMON_FOLLOW_INODE_CONFIG
+
+      d = create_driver(config, false)
+
+      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+        f.puts "test1"
+        f.puts "test2"
+      }
+
+      d.run(expect_emits: 3, shutdown: false) do
+        File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
+        sleep 2
+        File.open("#{TMP_DIR}/tail.txt", "w+b") {|f| f.puts "test4\n"}
+      end
+
+      events = d.events
+      assert_equal(4, events.length)
+      assert_equal({"message" => "test1"}, events[0][2])
+      assert_equal({"message" => "test2"}, events[1][2])
+      assert_equal({"message" => "test3"}, events[2][2])
+      assert_equal({"message" => "test4"}, events[3][2])
+      d.instance_shutdown
+    end
+  end
+
   sub_test_case "tail_path" do
     def test_tail_path_with_singleline
       File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
@@ -1452,13 +1895,13 @@ class TailInputTest < Test::Unit::TestCase
     })
 
     expected_files = [
-      "#{TMP_DIR}/tail_watch1.txt",
-      "#{TMP_DIR}/tail_watch2.txt"
+      create_target_info("#{TMP_DIR}/tail_watch1.txt"),
+      create_target_info("#{TMP_DIR}/tail_watch2.txt")
     ]
 
     Timecop.freeze(now) do
       plugin = create_driver(config, false).instance
-      assert_equal expected_files, plugin.expand_paths.sort
+      assert_equal expected_files, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
     end
   end
 
@@ -1476,4 +1919,22 @@ class TailInputTest < Test::Unit::TestCase
     waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size == 1 }
     d.instance_shutdown
   end
+
+  def test_ENOENT_error_after_setup_watcher
+    path = "#{TMP_DIR}/tail.txt"
+    FileUtils.touch(path)
+    config = config_element('', '', {
+                              'format' => 'none',
+                            })
+    d = create_driver(config)
+    mock.proxy(d.instance).setup_watcher(anything, anything) do |tw|
+      cleanup_file(path)
+      tw
+    end
+    assert_nothing_raised do
+      d.run(shutdown: false) {}
+    end
+    d.instance_shutdown
+    assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed with ENOENT. Drop tail watcher for now.\n") })
+  end
 end