fluentd 1.12.0.rc1 → 1.12.0.rc2

data/test/command/test_ctl.rb

@@ -0,0 +1,57 @@
+require_relative '../helper'
+
+require 'test-unit'
+require 'win32/event' if Fluent.windows?
+
+require 'fluent/command/ctl'
+
+class TestFluentdCtl < ::Test::Unit::TestCase
+  def assert_win32_event(event_name, command, pid_or_svcname)
+    command, event_suffix = data
+    event = Win32::Event.new(event_name)
+    ipc = Win32::Ipc.new(event.handle)
+    ret = Win32::Ipc::TIMEOUT
+
+    wait_thread = Thread.new do
+      ret = ipc.wait(1)
+    end
+    Fluent::Ctl.new([command, pid_or_svcname]).call
+    wait_thread.join
+    assert_equal(Win32::Ipc::SIGNALED, ret)
+  end
+
+  data("shutdown" => ["shutdown", "TERM", ""],
+       "restart" => ["restart", "HUP", "HUP"],
+       "flush" => ["flush", "USR1", "USR1"],
+       "reload" => ["reload", "USR2", "USR2"])
+  def test_commands(data)
+    command, signal, event_suffix = data
+
+    if Fluent.windows?
+      event_name = "fluentd_54321"
+      event_name << "_#{event_suffix}" unless event_suffix.empty?
+      assert_win32_event(event_name, command, "54321")
+    else
+      got_signal = false
+      Signal.trap(signal) do
+        got_signal = true
+      end
+      Fluent::Ctl.new([command, "#{$$}"]).call
+      assert_true(got_signal)
+    end
+  end
+
+  data("shutdown" => ["shutdown", ""],
+       "restart" => ["restart", "HUP"],
+       "flush" => ["flush", "USR1"],
+       "reload" => ["reload", "USR2"])
+  def test_commands_with_winsvcname(data)
+    omit "Only for Windows" unless Fluent.windows?
+
+    command, event_suffix = data
+    event_name = "testfluentdwinsvc"
+    event_name << "_#{event_suffix}" unless event_suffix.empty?
+
+    assert_win32_event(event_name, command, "testfluentdwinsvc")
+  end
+end

data/test/command/test_plugin_config_formatter.rb

@@ -56,6 +56,32 @@ class TestFluentPluginConfigFormatter < Test::Unit::TestCase
     end
   end
 
+  class FakeStorage < ::Fluent::Plugin::Storage
+    ::Fluent::Plugin.register_storage('fake', self)
+
+    def get(key)
+    end
+
+    def fetch(key, defval)
+    end
+
+    def put(key, value)
+    end
+
+    def delete(key)
+    end
+
+    def update(key, &block)
+    end
+  end
+
+  class FakeServiceDiscovery < ::Fluent::Plugin::ServiceDiscovery
+    ::Fluent::Plugin.register_sd('fake', self)
+
+    desc "hostname"
+    config_param :hostname, :string
+  end
+
   class SimpleInput < ::Fluent::Plugin::Input
     ::Fluent::Plugin.register_input("simple", self)
     helpers :inject, :compat_parameters
@@ -88,6 +114,13 @@ class TestFluentPluginConfigFormatter < Test::Unit::TestCase
     end
   end
 
+  class SimpleServiceDiscovery < ::Fluent::Plugin::ServiceDiscovery
+    ::Fluent::Plugin.register_sd('simple', self)
+
+    desc "servers"
+    config_param :servers, :array
+  end
+
   sub_test_case "json" do
     data(input: [FakeInput, "input"],
          output: [FakeOutput, "output"],
@@ -196,6 +229,28 @@ TEXT
       assert_equal(expected, dumped_config)
     end
 
+    data("abbrev" => "sd",
+         "normal" => "service_discovery")
+    test "service_discovery simple" do |data|
+      plugin_type = data
+      dumped_config = capture_stdout do
+        FluentPluginConfigFormatter.new(["--format=markdown", plugin_type, "simple"]).call
+      end
+      expected = <<TEXT
+* See also: [ServiceDiscovery Plugin Overview](https://docs.fluentd.org/v/1.0/servicediscovery#overview)
+
+## TestFluentPluginConfigFormatter::SimpleServiceDiscovery
+
+### servers (array) (required)
+
+servers
+
+
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+
     test "output complex" do
       dumped_config = capture_stdout do
         FluentPluginConfigFormatter.new(["--format=markdown", "output", "complex"]).call
@@ -251,7 +306,7 @@ TEXT
   sub_test_case "arguments" do
     data do
       hash = {}
-      ["input", "output", "filter", "parser", "formatter"].each do |type|
+      ["input", "output", "filter", "parser", "formatter", "storage", "service_discovery"].each do |type|
         ["txt", "json", "markdown"].each do |format|
           argv = ["--format=#{format}"]
           [
@@ -259,7 +314,7 @@ TEXT
             ["--verbose"],
             ["--compact"]
           ].each do |options|
-            hash[(argv + options).join(" ")] = argv + options + [type, "fake"]
+            hash["[#{type}] " + (argv + options).join(" ")] = argv + options + [type, "fake"]
           end
         end
       end

data/test/plugin/in_tail/test_position_file.rb

@@ -1,5 +1,6 @@
 require_relative '../../helper'
 require 'fluent/plugin/in_tail/position_file'
+require 'fluent/plugin/in_tail'
 
 require 'fileutils'
 require 'tempfile'
@@ -27,9 +28,15 @@ class IntailPositionFileTest < Test::Unit::TestCase
     f.seek(0)
   end
 
+  def follow_inodes_block
+    [true, false].each do |follow_inodes|
+      yield follow_inodes
+    end
+  end
+
   test '.load' do
     write_data(@file, TEST_CONTENT)
-    Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
+    Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
 
     @file.seek(0)
     lines = @file.readlines
@@ -41,7 +48,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
   sub_test_case '#try_compact' do
     test 'compact invalid and convert 32 bit inode value' do
       write_data(@file, TEST_CONTENT)
-      Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log).try_compact
+      Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log}).try_compact
 
       @file.seek(0)
       lines = @file.readlines
@@ -55,7 +62,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
         valid_path\t0000000000000002\t0000000000000001
         valid_path\t0000000000000003\t0000000000000004
       EOF
-      Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log).try_compact
+      Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log}).try_compact
 
       @file.seek(0)
       lines = @file.readlines
@@ -64,7 +71,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
 
     test 'does not change when the file is changed' do
       write_data(@file, TEST_CONTENT)
-      pf = Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log)
+      pf = Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log})
 
       mock.proxy(pf).fetch_compacted_entries do |r|
         @file.write("unwatched\t#{UNWATCHED_STR}\t0000000000000000\n")
@@ -79,11 +86,16 @@ class IntailPositionFileTest < Test::Unit::TestCase
     end
 
     test 'update seek postion of remained position entry' do
-      pf = Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log)
-      pf['path1']
-      pf['path2']
-      pf['path3']
-      pf.unwatch('path1')
+      pf = Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log})
+      target_info1 = Fluent::Plugin::TailInput::TargetInfo.new('path1', -1)
+      target_info2 = Fluent::Plugin::TailInput::TargetInfo.new('path2', -1)
+      target_info3 = Fluent::Plugin::TailInput::TargetInfo.new('path3', -1)
+      pf[target_info1]
+      pf[target_info2]
+      pf[target_info3]
+
+      target_info1_2 = Fluent::Plugin::TailInput::TargetInfo.new('path1', 1234)
+      pf.unwatch(target_info1_2)
 
       pf.try_compact
 
@@ -93,8 +105,10 @@ class IntailPositionFileTest < Test::Unit::TestCase
       assert_equal "path3\t0000000000000000\t0000000000000000\n", lines[1]
       assert_equal 2, lines.size
 
-      pf.unwatch('path2')
-      pf.unwatch('path3')
+      target_info2_2 = Fluent::Plugin::TailInput::TargetInfo.new('path2', 1235)
+      target_info3_2 = Fluent::Plugin::TailInput::TargetInfo.new('path3', 1236)
+      pf.unwatch(target_info2_2)
+      pf.unwatch(target_info3_2)
       @file.seek(0)
       lines = @file.readlines
       assert_equal "path2\t#{UNWATCHED_STR}\t0000000000000000\n", lines[0]
@@ -106,7 +120,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
   sub_test_case '#load' do
     test 'compact invalid and convert 32 bit inode value' do
       write_data(@file, TEST_CONTENT)
-      Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
+      Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
 
       @file.seek(0)
       lines = @file.readlines
@@ -120,7 +134,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
         valid_path\t0000000000000002\t0000000000000001
         valid_path\t0000000000000003\t0000000000000004
       EOF
-      Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log).load
+      Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log}).load
 
       @file.seek(0)
       lines = @file.readlines
@@ -131,9 +145,10 @@ class IntailPositionFileTest < Test::Unit::TestCase
   sub_test_case '#[]' do
     test 'return entry' do
       write_data(@file, TEST_CONTENT)
-      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
+      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
 
-      f = pf['valid_path']
+      valid_target_info = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', Fluent::FileWrapper.stat(@file).ino)
+      f = pf[valid_target_info]
       assert_equal Fluent::Plugin::TailInput::FilePositionEntry, f.class
       assert_equal 2, f.read_pos
       assert_equal 1, f.read_inode
@@ -142,7 +157,8 @@ class IntailPositionFileTest < Test::Unit::TestCase
       lines = @file.readlines
       assert_equal 2, lines.size
 
-      f = pf['nonexist_path']
+      nonexistent_target_info = Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)
+      f = pf[nonexistent_target_info]
       assert_equal Fluent::Plugin::TailInput::FilePositionEntry, f.class
       assert_equal 0, f.read_pos
       assert_equal 0, f.read_inode
@@ -155,19 +171,19 @@ class IntailPositionFileTest < Test::Unit::TestCase
 
     test 'does not change other value position if other entry try to write' do
       write_data(@file, TEST_CONTENT)
-      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
+      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, logger: $log)
 
-      f = pf['nonexist_path']
+      f = pf[Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)]
       assert_equal 0, f.read_inode
       assert_equal 0, f.read_pos
 
-      pf['valid_path'].update(1, 2)
+      pf[Fluent::Plugin::TailInput::TargetInfo.new('valid_path', Fluent::FileWrapper.stat(@file).ino)].update(1, 2)
 
-      f = pf['nonexist_path']
+      f = pf[Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)]
       assert_equal 0, f.read_inode
       assert_equal 0, f.read_pos
 
-      pf['nonexist_path'].update(1, 2)
+      pf[Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)].update(1, 2)
       assert_equal 1, f.read_inode
       assert_equal 2, f.read_pos
     end
@@ -176,14 +192,18 @@ class IntailPositionFileTest < Test::Unit::TestCase
   sub_test_case '#unwatch' do
     test 'deletes entry by path' do
       write_data(@file, TEST_CONTENT)
-      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
-      p1 = pf['valid_path']
+      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, logger: $log)
+      inode1 = Fluent::FileWrapper.stat(@file).ino
+      target_info1 = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', inode1)
+      p1 = pf[target_info1]
       assert_equal Fluent::Plugin::TailInput::FilePositionEntry, p1.class
 
-      pf.unwatch('valid_path')
+      pf.unwatch(target_info1)
       assert_equal p1.read_pos, Fluent::Plugin::TailInput::PositionFile::UNWATCHED_POSITION
 
-      p2 = pf['valid_path']
+      inode2 = Fluent::FileWrapper.stat(@file).ino
+      target_info2 = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', inode2)
+      p2 = pf[target_info2]
       assert_equal Fluent::Plugin::TailInput::FilePositionEntry, p2.class
 
       assert_not_equal p1, p2

data/test/plugin/test_in_http.rb

@@ -46,6 +46,7 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal 10*1024*1024, d.instance.body_size_limit
     assert_equal 5, d.instance.keepalive_timeout
     assert_equal false, d.instance.add_http_headers
+    assert_equal false, d.instance.add_query_params
   end
 
   def test_time
@@ -907,6 +908,30 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal ["403", "403"], res_codes
   end
 
+  def test_add_query_params 
+    d = create_driver(CONFIG + "add_query_params true")
+    assert_equal true, d.instance.add_query_params
+
+    time = event_time("2011-01-02 13:14:15 UTC")
+    time_i = time.to_i
+    events = [
+      ["tag1", time, {"a"=>1, "QUERY_A"=>"b"}],
+      ["tag2", time, {"a"=>2, "QUERY_A"=>"b"}],
+    ]
+    res_codes = []
+    res_bodies = []
+
+    d.run do
+      events.each do |tag, _t, record|
+        res = post("/#{tag}?a=b", {"json"=>record.to_json, "time"=>time_i.to_s})
+        res_codes << res.code
+      end
+    end
+    assert_equal ["200", "200"], res_codes
+    assert_equal [], res_bodies
+    assert_equal events, d.events
+  end
+
   $test_in_http_connection_object_ids = []
   $test_in_http_content_types = []
   $test_in_http_content_types_flag = false

data/test/plugin/test_in_tail.rb

@@ -17,29 +17,62 @@ class TailInputTest < Test::Unit::TestCase
 
   def teardown
     super
+    cleanup_directory(TMP_DIR)
     Fluent::Engine.stop
   end
 
   def cleanup_directory(path)
-    FileUtils.rm_rf(path, secure: true)
+    begin
+      FileUtils.rm_f(path, secure: true)
+    rescue ArgumentError
+      FileUtils.rm_f(path) # For Ruby 2.6 or before.
+    end
     if File.exist?(path)
       FileUtils.remove_entry_secure(path, true)
     end
     FileUtils.mkdir_p(path)
   end
 
+  def cleanup_file(path)
+    begin
+      FileUtils.rm_f(path, secure: true)
+    rescue ArgumentError
+      FileUtils.rm_f(path) # For Ruby 2.6 or before.
+    end
+    if File.exist?(path)
+      # ensure files are closed for Windows, on which deleted files
+      # are still visible from filesystem
+      GC.start(full_mark: true, immediate_mark: true, immediate_sweep: true)
+      FileUtils.remove_entry_secure(path, true)
+    end
+  end
+
+  def create_target_info(path)
+    Fluent::Plugin::TailInput::TargetInfo.new(path, Fluent::FileWrapper.stat(path).ino)
+  end
+
   TMP_DIR = File.dirname(__FILE__) + "/../tmp/tail#{ENV['TEST_ENV_NUMBER']}"
 
   CONFIG = config_element("ROOT", "", {
                             "path" => "#{TMP_DIR}/tail.txt",
                             "tag" => "t1",
-                            "rotate_wait" => "2s"
+                            "rotate_wait" => "2s",
+                            "refresh_interval" => "1s"
                           })
   COMMON_CONFIG = CONFIG + config_element("", "", { "pos_file" => "#{TMP_DIR}/tail.pos" })
   CONFIG_READ_FROM_HEAD = config_element("", "", { "read_from_head" => true })
   CONFIG_ENABLE_WATCH_TIMER = config_element("", "", { "enable_watch_timer" => false })
   CONFIG_DISABLE_STAT_WATCHER = config_element("", "", { "enable_stat_watcher" => false })
   CONFIG_OPEN_ON_EVERY_UPDATE = config_element("", "", { "open_on_every_update" => true })
+  COMMON_FOLLOW_INODE_CONFIG = config_element("ROOT", "", {
+                                                "path" => "#{TMP_DIR}/tail.txt*",
+                                                "pos_file" => "#{TMP_DIR}/tail.pos",
+                                                "tag" => "t1",
+                                                "refresh_interval" => "1s",
+                                                "read_from_head" => "true",
+                                                "format" => "none",
+                                                "follow_inodes" => "true"
+                                              })
   SINGLE_LINE_CONFIG = config_element("", "", { "format" => "/(?<message>.*)/" })
   PARSE_SINGLE_LINE_CONFIG = config_element("", "", {}, [config_element("parse", "", { "@type" => "/(?<message>.*)/" })])
   MULTILINE_CONFIG = config_element(
@@ -118,6 +151,12 @@ class TailInputTest < Test::Unit::TestCase
       end
     end
 
+    test "follow_inodes w/o pos file" do
+      assert_raise(Fluent::ConfigError) do
+        create_driver(CONFIG + config_element('', '', {'follow_inodes' => 'true'}))
+      end
+    end
+
     test "both enable_watch_timer and enable_stat_watcher are false" do
       assert_raise(Fluent::ConfigError) do
         create_driver(CONFIG_ENABLE_WATCH_TIMER + CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)
@@ -1002,6 +1041,7 @@ class TailInputTest < Test::Unit::TestCase
     # * path test
     # TODO: Clean up tests
     EX_ROTATE_WAIT = 0
+    EX_FOLLOW_INODES = false
 
     EX_CONFIG = config_element("", "", {
                                  "tag" => "tail",
@@ -1011,38 +1051,43 @@ class TailInputTest < Test::Unit::TestCase
                                  "read_from_head" => true,
                                  "refresh_interval" => 30,
                                  "rotate_wait" => "#{EX_ROTATE_WAIT}s",
+                                 "follow_inodes" => "#{EX_FOLLOW_INODES}",
                                })
-    EX_PATHS = [
-      'test/plugin/data/2010/01/20100102-030405.log',
-      'test/plugin/data/log/foo/bar.log',
-      'test/plugin/data/log/test.log'
-    ]
-
     def test_expand_paths
+      ex_paths = [
+        create_target_info('test/plugin/data/2010/01/20100102-030405.log'),
+        create_target_info('test/plugin/data/log/foo/bar.log'),
+        create_target_info('test/plugin/data/log/test.log')
+      ]
       plugin = create_driver(EX_CONFIG, false).instance
       flexstub(Time) do |timeclass|
         timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 2, 3, 4, 5))
-        assert_equal EX_PATHS, plugin.expand_paths.sort
+        assert_equal ex_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
       end
 
       # Test exclusion
-      exclude_config = EX_CONFIG + config_element("", "", { "exclude_path" => %Q(["#{EX_PATHS.last}"]) })
+      exclude_config = EX_CONFIG + config_element("", "", { "exclude_path" => %Q(["#{ex_paths.last.path}"]) })
       plugin = create_driver(exclude_config, false).instance
-      assert_equal EX_PATHS - [EX_PATHS.last], plugin.expand_paths.sort
+      assert_equal ex_paths - [ex_paths.last], plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
     end
 
     def test_expand_paths_with_duplicate_configuration
       expanded_paths = [
-        'test/plugin/data/log/foo/bar.log',
-        'test/plugin/data/log/test.log'
+        create_target_info('test/plugin/data/log/foo/bar.log'),
+        create_target_info('test/plugin/data/log/test.log')
       ]
       duplicate_config = EX_CONFIG.dup
       duplicate_config["path"]="test/plugin/data/log/**/*.log, test/plugin/data/log/**/*.log"
       plugin = create_driver(EX_CONFIG, false).instance
-      assert_equal expanded_paths, plugin.expand_paths.sort
+      assert_equal expanded_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
     end
 
     def test_expand_paths_with_timezone
+      ex_paths = [
+        create_target_info('test/plugin/data/2010/01/20100102-030405.log'),
+        create_target_info('test/plugin/data/log/foo/bar.log'),
+        create_target_info('test/plugin/data/log/test.log')
+      ]
       ['Asia/Taipei', '+08'].each do |tz_type|
         taipei_config = EX_CONFIG + config_element("", "", {"path_timezone" => tz_type})
         plugin = create_driver(taipei_config, false).instance
@@ -1056,8 +1101,8 @@ class TailInputTest < Test::Unit::TestCase
             # env : 2010-01-01 19:04:05 (UTC), tail path : 2010-01-02 03:04:05 (Asia/Taipei)
             timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 1, 19, 4, 5))
 
-            assert_equal EX_PATHS, plugin.expand_paths.sort
-            assert_equal EX_PATHS - [EX_PATHS.first], exclude_plugin.expand_paths.sort
+            assert_equal ex_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
+            assert_equal ex_paths - [ex_paths.first], exclude_plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
           end
         end
       end
@@ -1065,10 +1110,10 @@ class TailInputTest < Test::Unit::TestCase
 
     def test_log_file_without_extension
       expected_files = [
-        'test/plugin/data/log/bar',
-        'test/plugin/data/log/foo/bar.log',
-        'test/plugin/data/log/foo/bar2',
-        'test/plugin/data/log/test.log'
+        create_target_info('test/plugin/data/log/bar'),
+        create_target_info('test/plugin/data/log/foo/bar.log'),
+        create_target_info('test/plugin/data/log/foo/bar2'),
+        create_target_info('test/plugin/data/log/test.log')
       ]
 
       config = config_element("", "", {
@@ -1079,7 +1124,7 @@ class TailInputTest < Test::Unit::TestCase
       })
 
       plugin = create_driver(config, false).instance
-      assert_equal expected_files, plugin.expand_paths.sort
+      assert_equal expected_files, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
     end
 
     def test_unwatched_files_should_be_removed
@@ -1205,25 +1250,35 @@ class TailInputTest < Test::Unit::TestCase
   end
 
   def test_z_refresh_watchers
+    ex_paths = [
+      create_target_info('test/plugin/data/2010/01/20100102-030405.log'),
+      create_target_info('test/plugin/data/log/foo/bar.log'),
+      create_target_info('test/plugin/data/log/test.log'),
+    ]
     plugin = create_driver(EX_CONFIG, false).instance
     sio = StringIO.new
     plugin.instance_eval do
-      @pf = Fluent::Plugin::TailInput::PositionFile.load(sio, logger: $log)
+      @pf = Fluent::Plugin::TailInput::PositionFile.load(sio, EX_FOLLOW_INODES, {}, logger: $log)
       @loop = Coolio::Loop.new
     end
 
     Timecop.freeze(2010, 1, 2, 3, 4, 5) do
-      EX_PATHS.each do |path|
-        mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(path, anything, anything, true, anything, nil, anything).once
+      ex_paths.each do |target_info|
+        mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything).once
       end
 
       plugin.refresh_watchers
     end
 
-    mock.proxy(plugin).detach_watcher_after_rotate_wait(plugin.instance_variable_get(:@tails)['test/plugin/data/2010/01/20100102-030405.log'])
+    path = 'test/plugin/data/2010/01/20100102-030405.log'
+    target_info = Fluent::Plugin::TailInput::TargetInfo.new(path, Fluent::FileWrapper.stat(path).ino)
+    mock.proxy(plugin).detach_watcher_after_rotate_wait(plugin.instance_variable_get(:@tails)[target_info], target_info.ino)
 
     Timecop.freeze(2010, 1, 2, 3, 4, 6) do
-      mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new('test/plugin/data/2010/01/20100102-030406.log', anything, anything, true, anything, nil, anything).once
+      path = "test/plugin/data/2010/01/20100102-030406.log"
+      inode = Fluent::FileWrapper.stat(path).ino
+      target_info = Fluent::Plugin::TailInput::TargetInfo.new(path, inode)
+      mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything).once
       plugin.refresh_watchers
 
       flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
@@ -1369,6 +1424,305 @@ class TailInputTest < Test::Unit::TestCase
     end
   end
 
+  sub_test_case 'inode_processing' do
+    def test_should_delete_file_pos_entry_for_non_existing_file_with_follow_inodes
+      config = COMMON_FOLLOW_INODE_CONFIG
+
+      path = "#{TMP_DIR}/tail.txt"
+      ino = 1
+      pos = 1234
+      File.open("#{TMP_DIR}/tail.pos", "wb") {|f|
+        f.puts ("%s\t%016x\t%016x\n" % [path, pos, ino])
+      }
+
+      d = create_driver(config, false)
+      d.run
+
+      pos_file = File.open("#{TMP_DIR}/tail.pos", "r")
+      pos_file.pos = 0
+
+      assert_raise(EOFError) do
+        pos_file.readline
+      end
+    end
+
+    def test_should_write_latest_offset_after_rotate_wait
+      config = COMMON_FOLLOW_INODE_CONFIG
+      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+        f.puts "test1"
+        f.puts "test2"
+      }
+
+      d = create_driver(config, false)
+      d.run(expect_emits: 2, shutdown: false) do
+        File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
+        FileUtils.move("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail.txt" + "1")
+        sleep 1
+        File.open("#{TMP_DIR}/tail.txt" + "1", "ab") {|f| f.puts "test4\n"}
+      end
+
+      pos_file = File.open("#{TMP_DIR}/tail.pos", "r")
+      pos_file.pos = 0
+      line_parts = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
+      waiting(5) {
+        while line_parts[2].to_i(16) != 24
+          sleep(0.1)
+          pos_file.pos = 0
+          line_parts = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
+        end
+      }
+      assert_equal(24, line_parts[2].to_i(16))
+      d.instance_shutdown
+    end
+
+    def test_should_keep_and_update_existing_file_pos_entry_for_deleted_file_when_new_file_with_same_name_created
+      config = config_element("", "", {"format" => "none"})
+
+      path = "#{TMP_DIR}/tail.txt"
+      ino = 1
+      pos = 1234
+      File.open("#{TMP_DIR}/tail.pos", "wb") {|f|
+        f.puts ("%s\t%016x\t%016x\n" % [path, pos, ino])
+      }
+
+      d = create_driver(config)
+      d.run(shutdown: false)
+
+      pos_file = File.open("#{TMP_DIR}/tail.pos", "r")
+      pos_file.pos = 0
+
+      path_pos_ino = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
+      assert_equal(path, path_pos_ino[1])
+      assert_equal(pos, path_pos_ino[2].to_i(16))
+      assert_equal(ino, path_pos_ino[3].to_i(16))
+
+      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+        f.puts "test1"
+        f.puts "test2"
+      }
+      Timecop.travel(Time.now + 10) do
+        sleep 5
+        pos_file.pos = 0
+        tuple = create_target_info("#{TMP_DIR}/tail.txt")
+        path_pos_ino = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
+        assert_equal(tuple.path, path_pos_ino[1])
+        assert_equal(12, path_pos_ino[2].to_i(16))
+        assert_equal(tuple.ino, path_pos_ino[3].to_i(16))
+      end
+      d.instance_shutdown
+    end
+
+    def test_should_mark_file_unwatched_after_limit_recently_modified_and_rotate_wait
+      config = config_element("ROOT", "", {
+          "path" => "#{TMP_DIR}/tail.txt*",
+          "pos_file" => "#{TMP_DIR}/tail.pos",
+          "tag" => "t1",
+          "rotate_wait" => "1s",
+          "refresh_interval" => "1s",
+          "limit_recently_modified" => "1s",
+          "read_from_head" => "true",
+          "format" => "none",
+          "follow_inodes" => "true",
+      })
+
+      d = create_driver(config, false)
+
+      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+        f.puts "test1"
+        f.puts "test2"
+      }
+      target_info = create_target_info("#{TMP_DIR}/tail.txt")
+
+      d.run(expect_emits: 1, shutdown: false) do
+        File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
+      end
+
+
+      Timecop.travel(Time.now + 10) do
+        waiting(5) {
+          # @pos will be reset as 0 when UNWATCHED_POSITION is specified.
+          sleep 0.1 until d.instance.instance_variable_get(:@pf)[target_info].read_pos == 0
+        }
+      end
+
+      assert_equal(0, d.instance.instance_variable_get(:@pf)[target_info].read_pos)
+
+      d.instance_shutdown
+    end
+
+    def test_should_read_from_head_on_file_renaming_with_star_in_pattern
+      config = config_element("ROOT", "", {
+          "path" => "#{TMP_DIR}/tail.txt*",
+          "pos_file" => "#{TMP_DIR}/tail.pos",
+          "tag" => "t1",
+          "rotate_wait" => "10s",
+          "refresh_interval" => "1s",
+          "limit_recently_modified" => "60s",
+          "read_from_head" => "true",
+          "format" => "none",
+          "follow_inodes" => "true"
+      })
+
+      d = create_driver(config, false)
+
+      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+        f.puts "test1"
+        f.puts "test2"
+      }
+
+      d.run(expect_emits: 2, shutdown: false) do
+        File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
+        FileUtils.move("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail.txt1")
+      end
+
+      events = d.events
+      assert_equal(3, events.length)
+      d.instance_shutdown
+    end
+
+    def test_should_not_read_from_head_on_rotation_when_watching_inodes
+      config = COMMON_FOLLOW_INODE_CONFIG
+
+      d = create_driver(config, false)
+
+      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+        f.puts "test1"
+        f.puts "test2"
+      }
+
+      d.run(expect_emits: 1, shutdown: false) do
+        File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
+      end
+
+      FileUtils.move("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail.txt1")
+      Timecop.travel(Time.now + 10) do
+        sleep 2
+        events = d.events
+        assert_equal(3, events.length)
+      end
+
+      d.instance_shutdown
+    end
+
+    def test_should_mark_file_unwatched_if_same_name_file_created_with_different_inode
+      config = COMMON_FOLLOW_INODE_CONFIG
+
+      d = create_driver(config, false)
+
+      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+        f.puts "test1"
+        f.puts "test2"
+      }
+      target_info = create_target_info("#{TMP_DIR}/tail.txt")
+
+      d.run(expect_emits: 2, shutdown: false) do
+        File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
+        cleanup_file("#{TMP_DIR}/tail.txt")
+        File.open("#{TMP_DIR}/tail.txt", "wb") {|f| f.puts "test4\n"}
+      end
+
+      new_target_info = create_target_info("#{TMP_DIR}/tail.txt")
+
+      pos_file = d.instance.instance_variable_get(:@pf)
+
+      waiting(10) {
+        # @pos will be reset as 0 when UNWATCHED_POSITION is specified.
+        sleep 0.1 until pos_file[target_info].read_pos == 0
+      }
+      new_position = pos_file[new_target_info].read_pos
+      assert_equal(6, new_position)
+
+      d.instance_shutdown
+    end
+
+    def test_should_close_watcher_after_rotate_wait
+      now = Time.now
+      config = COMMON_FOLLOW_INODE_CONFIG + config_element('', '', {"rotate_wait" => "1s", "limit_recently_modified" => "1s"})
+
+      d = create_driver(config, false)
+
+      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+        f.puts "test1"
+        f.puts "test2"
+      }
+      target_info = create_target_info("#{TMP_DIR}/tail.txt")
+      mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, true, anything, nil, anything).once
+      d.run(shutdown: false)
+      assert d.instance.instance_variable_get(:@tails)[target_info]
+
+      Timecop.travel(now + 10) do
+        d.instance.instance_eval do
+          sleep 0.1 until @tails[target_info] == nil
+        end
+        assert_nil d.instance.instance_variable_get(:@tails)[target_info]
+      end
+      d.instance_shutdown
+    end
+
+    def test_should_create_new_watcher_for_new_file_with_same_name
+      now = Time.now
+      config = COMMON_FOLLOW_INODE_CONFIG + config_element('', '', {"limit_recently_modified" => "2s"})
+
+      d = create_driver(config, false)
+
+      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+        f.puts "test1"
+        f.puts "test2"
+      }
+      path_ino = create_target_info("#{TMP_DIR}/tail.txt")
+
+      d.run(expect_emits: 1, shutdown: false) do
+        File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
+      end
+
+      cleanup_file("#{TMP_DIR}/tail.txt")
+      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+        f.puts "test3"
+        f.puts "test4"
+      }
+      new_path_ino = create_target_info("#{TMP_DIR}/tail.txt")
+
+      Timecop.travel(now + 10) do
+        sleep 3
+        d.instance.instance_eval do
+          @tails[path_ino] == nil
+          @tails[new_path_ino] != nil
+        end
+      end
+
+      events = d.events
+
+      assert_equal(5, events.length)
+
+      d.instance_shutdown
+    end
+
+    def test_truncate_file_with_follow_inodes
+      config = COMMON_FOLLOW_INODE_CONFIG
+
+      d = create_driver(config, false)
+
+      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+        f.puts "test1"
+        f.puts "test2"
+      }
+
+      d.run(expect_emits: 3, shutdown: false) do
+        File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3\n"}
+        sleep 2
+        File.open("#{TMP_DIR}/tail.txt", "w+b") {|f| f.puts "test4\n"}
+      end
+
+      events = d.events
+      assert_equal(4, events.length)
+      assert_equal({"message" => "test1"}, events[0][2])
+      assert_equal({"message" => "test2"}, events[1][2])
+      assert_equal({"message" => "test3"}, events[2][2])
+      assert_equal({"message" => "test4"}, events[3][2])
+      d.instance_shutdown
+    end
+  end
+
   sub_test_case "tail_path" do
     def test_tail_path_with_singleline
       File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
@@ -1498,13 +1852,13 @@ class TailInputTest < Test::Unit::TestCase
     })
 
     expected_files = [
-      "#{TMP_DIR}/tail_watch1.txt",
-      "#{TMP_DIR}/tail_watch2.txt"
+      create_target_info("#{TMP_DIR}/tail_watch1.txt"),
+      create_target_info("#{TMP_DIR}/tail_watch2.txt")
     ]
 
     Timecop.freeze(now) do
       plugin = create_driver(config, false).instance
-      assert_equal expected_files, plugin.expand_paths.sort
+      assert_equal expected_files, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path }
     end
   end