fluentd 1.13.2 → 1.13.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b2f70ac4c7ffec54701aaf1e48416d2806881d39171a63c2eb84f3686a59cd59
-  data.tar.gz: b192b8229744e5fc00816c45b6a6e83bb1748d0eae1773394044907254fa6433
+  metadata.gz: 1603571f755a962fde001cbb33fd24ad8929633bdeb38832e61f45da2c0dfba5
+  data.tar.gz: 5c2d45299617e9e4424758c76eb53d78889387cedc28d1d8bac549402094b6af
 SHA512:
-  metadata.gz: 338cc5d35a89c042fdfa1a1817479fbfc2b88363619a5cb0cdf08a20a70942c52a561c538e70fb3b737eb7fa2d38bb77a43e993b3693b87bd300b43d64f8ac0c
-  data.tar.gz: aaace3c4450de3c4db7b6875e94e47b30e265f7370fcfcff7e15b9c4568913643fa53e3564e7b61237091e0bbf02fac21bdca93181c4109949e384b73fd072a1
+  metadata.gz: 8dd4cc4947766345b5455fbae8043610d1c3d33e0e2ea9ea348a866722c099545301236d507d754e31b063017bb7daefdc8aef0e2a8078b67df83eb3c2489b2c
+  data.tar.gz: f0d126ed7704b519c2b7db5693de9685db7fd3ed1844f770a08e1e3351f3daf0f66e4b88c9435b3e456f40e38480c9af7652241f029a36b7a13184364a595251

data/CHANGELOG.md

@@ -1,3 +1,27 @@
+# v1.13.3
+
+## Release v1.13.3 - 2021/07/27
+
+### Bug fix
+
+* in_tail: Care DeletePending state on Windows
+  https://github.com/fluent/fluentd/pull/3457
+  https://github.com/fluent/fluentd/pull/3460
+* in_tail: Fix some pos_file bugs.
+  Avoid deleting pos_file entries unexpectedly when both
+  `pos_file_compaction_interval` and `follow_inode` are enabled.
+  Use `bytesize` instead of `size` for path length.
+  https://github.com/fluent/fluentd/pull/3459
+* in_tail: Fix detecting rotation twice on `follow_inode`.
+  https://github.com/fluent/fluentd/pull/3466
+
+### Misc
+
+* Remove needless spaces in a sample config file
+  https://github.com/fluent/fluentd/pull/3456
+
+### Enhancement
+
 # v1.13.2
 
 ## Release v1.13.2 - 2021/07/12

data/example/v0_12_filter.conf

@@ -52,10 +52,10 @@
 </match>
 
 # Composing two filters. For all events with the tag foo.bar,
-# 
+#
 # 1. The first filter filters out all events that has the field "hello"
 # 2. Then, for those events, we downcase the value of the "name" field.
-# 
+#
 # - {"name":"SADA", "hello":100} gets filtered out
 # - {"name":"SADA"} becomes {"name":"sada"}
 # - {"last_name":"FURUHASHI"} throws an error because it has no field called "name"

data/lib/fluent/plugin/file_wrapper.rb

@@ -155,7 +155,29 @@ module Fluent
       by_handle_file_information.unpack("I11Q1")[11] # fileindex
     end
 
+    # DeletePending is a Windows-specific file state that roughly means
+    # "this file is queued for deletion, so close any open handlers"
+    #
+    # This flag can be retrieved via GetFileInformationByHandleEx().
+    #
+    # https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-getfileinformationbyhandleex
+    #
+    def delete_pending
+      file_standard_info = 0x01
+      bufsize = 1024
+      buf = '\0' * bufsize
+
+      unless GetFileInformationByHandleEx.call(@file_handle, file_standard_info, buf, bufsize)
+        return false
+      end
+
+      return buf.unpack("QQICC")[3] != 0
+    end
+
+    private :delete_pending
+
     def stat
+      raise Errno::ENOENT if delete_pending
       s = File.stat(@path)
       s.instance_variable_set :@ino, self.ino
       def s.ino; @ino; end

data/lib/fluent/plugin/in_tail.rb

@@ -426,6 +426,7 @@ module Fluent::Plugin
 
       begin
         target_info = TargetInfo.new(target_info.path, Fluent::FileWrapper.stat(target_info.path).ino)
+        @tails.delete(target_info)
         @tails[target_info] = tw
         tw.on_notify
       rescue Errno::ENOENT, Errno::EACCES => e
@@ -491,10 +492,17 @@ module Fluent::Plugin
         new_position_entry = @pf[target_info]
 
         if new_position_entry.read_inode == 0
+          # When follow_inodes is true, it's not cleaned up by refresh_watcher.
+          # So it should be unwatched here explicitly.
+          rotated_tw.unwatched = true
+          # Make sure to delete old key, it has a different ino while the hash key is same.
+          @tails.delete(rotated_target_info)
           @tails[new_target_info] = setup_watcher(new_target_info, new_position_entry)
           @tails[new_target_info].on_notify
         end
       else
+        # Make sure to delete old key, it has a different ino while the hash key is same.
+        @tails.delete(rotated_target_info)
         @tails[new_target_info] = setup_watcher(new_target_info, pe)
         @tails[new_target_info].on_notify
       end
@@ -831,7 +839,7 @@ module Fluent::Plugin
               # new watcher, and old watcher will be closed by stop_watcher in refresh_watchers method
               # don't want to swap state because we need latest read offset in pos file even after rotate_wait
               if stat
-                target_info = TargetInfo.new(@path, stat)
+                target_info = TargetInfo.new(@path, stat.ino)
                 @update_watcher.call(target_info, @pe)
               end
             else

data/lib/fluent/plugin/in_tail/position_file.rb

@@ -22,19 +22,18 @@ module Fluent::Plugin
       UNWATCHED_POSITION = 0xffffffffffffffff
       POSITION_FILE_ENTRY_REGEX = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.freeze
 
-      def self.load(file, follow_inodes, existing_paths, logger:)
-        pf = new(file, follow_inodes, existing_paths, logger: logger)
-        pf.load
+      def self.load(file, follow_inodes, existing_targets, logger:)
+        pf = new(file, follow_inodes, logger: logger)
+        pf.load(existing_targets)
         pf
       end
 
-      def initialize(file, follow_inodes, existing_paths, logger: nil)
+      def initialize(file, follow_inodes, logger: nil)
         @file = file
         @logger = logger
         @file_mutex = Mutex.new
         @map = {}
         @follow_inodes = follow_inodes
-        @existing_paths = existing_paths
       end
 
       def [](target_info)
@@ -60,8 +59,8 @@ module Fluent::Plugin
         end
       end
 
-      def load
-        compact
+      def load(existing_targets = nil)
+        compact(existing_targets)
 
         map = {}
         @file_mutex.synchronize do
@@ -118,9 +117,9 @@ module Fluent::Plugin
 
       private
 
-      def compact
+      def compact(existing_targets = nil)
         @file_mutex.synchronize do
-          entries = fetch_compacted_entries.values.map(&:to_entry_fmt)
+          entries = fetch_compacted_entries(existing_targets).values.map(&:to_entry_fmt)
 
           @file.pos = 0
           @file.truncate(0)
@@ -128,7 +127,7 @@ module Fluent::Plugin
         end
       end
 
-      def fetch_compacted_entries
+      def fetch_compacted_entries(existing_targets = nil)
         entries = {}
 
         @file.pos = 0
@@ -151,23 +150,26 @@ module Fluent::Plugin
             end
 
             if @follow_inodes
-              entries[ino] = Entry.new(path, pos, ino, file_pos + path.size + 1)
+              entries[ino] = Entry.new(path, pos, ino, file_pos + path.bytesize + 1)
             else
-              entries[path] = Entry.new(path, pos, ino, file_pos + path.size + 1)
+              entries[path] = Entry.new(path, pos, ino, file_pos + path.bytesize + 1)
             end
             file_pos += line.size
           end
         end
 
-        entries = remove_deleted_files_entries(entries, @existing_paths) if @follow_inodes
+        entries = remove_deleted_files_entries(entries, existing_targets) if @follow_inodes
         entries
       end
 
-      def remove_deleted_files_entries(existent_entries, existing_paths)
-        filtered_entries = existent_entries.select {|file_entry|
-          existing_paths.key?(file_entry)
-        }
-        filtered_entries
+      def remove_deleted_files_entries(existent_entries, existing_targets)
+        if existing_targets
+          existent_entries.select { |path_or_ino|
+            existing_targets.key?(path_or_ino)
+          }
+        else
+          existent_entries
+        end
       end
     end
 

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.13.2'
+  VERSION = '1.13.3'
 
 end

data/test/plugin/in_tail/test_position_file.rb

@@ -48,7 +48,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
   sub_test_case '#try_compact' do
     test 'compact invalid and convert 32 bit inode value' do
       write_data(@file, TEST_CONTENT)
-      Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log}).try_compact
+      Fluent::Plugin::TailInput::PositionFile.new(@file, false, **{logger: $log}).try_compact
 
       @file.seek(0)
       lines = @file.readlines
@@ -62,7 +62,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
         valid_path\t0000000000000002\t0000000000000001
         valid_path\t0000000000000003\t0000000000000004
       EOF
-      Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log}).try_compact
+      Fluent::Plugin::TailInput::PositionFile.new(@file, false, **{logger: $log}).try_compact
 
       @file.seek(0)
       lines = @file.readlines
@@ -71,7 +71,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
 
     test 'does not change when the file is changed' do
       write_data(@file, TEST_CONTENT)
-      pf = Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log})
+      pf = Fluent::Plugin::TailInput::PositionFile.new(@file, false, **{logger: $log})
 
       mock.proxy(pf).fetch_compacted_entries do |r|
         @file.write("unwatched\t#{UNWATCHED_STR}\t0000000000000000\n")
@@ -86,7 +86,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
     end
 
     test 'update seek position of remained position entry' do
-      pf = Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log})
+      pf = Fluent::Plugin::TailInput::PositionFile.new(@file, false, **{logger: $log})
       target_info1 = Fluent::Plugin::TailInput::TargetInfo.new('path1', -1)
       target_info2 = Fluent::Plugin::TailInput::TargetInfo.new('path2', -1)
       target_info3 = Fluent::Plugin::TailInput::TargetInfo.new('path3', -1)
@@ -115,6 +115,24 @@ class IntailPositionFileTest < Test::Unit::TestCase
       assert_equal "path3\t#{UNWATCHED_STR}\t0000000000000000\n", lines[1]
       assert_equal 2, lines.size
     end
+
+    test 'should ignore initial existing files on follow_inode' do
+      write_data(@file, TEST_CONTENT)
+      pos_file = Fluent::Plugin::TailInput::PositionFile.load(@file, true, {}, **{logger: $log})
+      @file.seek(0)
+      assert_equal([], @file.readlines)
+
+      @file.seek(0)
+      write_data(@file, TEST_CONTENT)
+      pos_file.try_compact
+
+      @file.seek(0)
+      assert_equal([
+                     "valid_path\t0000000000000002\t0000000000000001\n",
+                     "inode23bit\t0000000000000000\t0000000000000000\n",
+                   ],
+                   @file.readlines)
+    end
   end
 
   sub_test_case '#load' do
@@ -134,7 +152,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
         valid_path\t0000000000000002\t0000000000000001
         valid_path\t0000000000000003\t0000000000000004
       EOF
-      Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log}).load
+      Fluent::Plugin::TailInput::PositionFile.new(@file, false, **{logger: $log}).load
 
       @file.seek(0)
       lines = @file.readlines

data/test/plugin/test_file_wrapper.rb

@@ -96,6 +96,17 @@ class FileWrapperTest < Test::Unit::TestCase
       end
     end
 
+    test 'Errno::ENOENT raised on DeletePending' do
+      path = "#{TMP_DIR}/deletepending.txt"
+      file = Fluent::WindowsFile.new(path, mode='w')
+      File.delete(path)
+      assert_raise(Errno::ENOENT) do
+        file.stat
+      ensure
+        file.close if file
+      end
+    end
+
     test 'ERROR_SHARING_VIOLATION raised' do
       begin
         path = "#{TMP_DIR}/test_windows_file.txt"

data/test/plugin/test_in_tail.rb

@@ -109,6 +109,7 @@ class TailInputTest < Test::Unit::TestCase
                                                 "refresh_interval" => "1s",
                                                 "read_from_head" => "true",
                                                 "format" => "none",
+                                                "rotate_wait" => "1s",
                                                 "follow_inodes" => "true"
                                               })
   SINGLE_LINE_CONFIG = config_element("", "", { "format" => "/(?<message>.*)/" })
@@ -1984,6 +1985,49 @@ class TailInputTest < Test::Unit::TestCase
       assert_equal({"message" => "test4"}, events[3][2])
       d.instance_shutdown
     end
+
+    # issue #3464
+    def test_should_replace_target_info
+      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+        f.puts "test1\n"
+      }
+      target_info = create_target_info("#{TMP_DIR}/tail.txt")
+      inodes = []
+
+      config = config_element("ROOT", "", {
+                                "path" => "#{TMP_DIR}/tail.txt*",
+                                "pos_file" => "#{TMP_DIR}/tail.pos",
+                                "tag" => "t1",
+                                "refresh_interval" => "60s",
+                                "read_from_head" => "true",
+                                "format" => "none",
+                                "rotate_wait" => "1s",
+                                "follow_inodes" => "true"
+                              })
+      d = create_driver(config, false)
+      d.run(timeout: 5) do
+        while d.events.size < 1 do
+          sleep 0.1
+        end
+        inodes = d.instance.instance_variable_get(:@tails).keys.collect do |key|
+          key.ino
+        end
+        assert_equal([target_info.ino], inodes)
+
+        cleanup_file("#{TMP_DIR}/tail.txt")
+        File.open("#{TMP_DIR}/tail.txt", "wb") {|f| f.puts "test2\n"}
+
+        while d.events.size < 2 do
+          sleep 0.1
+        end
+        inodes = d.instance.instance_variable_get(:@tails).keys.collect do |key|
+          key.ino
+        end
+        new_target_info = create_target_info("#{TMP_DIR}/tail.txt")
+        assert_not_equal(target_info.ino, new_target_info.ino)
+        assert_equal([new_target_info.ino], inodes)
+      end
+    end
   end
 
   sub_test_case "tail_path" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.13.2
+  version: 1.13.3
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-07-12 00:00:00.000000000 Z
+date: 2021-07-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler