fluentd 1.16.1-x64-mingw32 → 1.16.3-x64-mingw32

data/lib/fluent/plugin/in_tail.rb

@@ -370,17 +370,52 @@ module Fluent::Plugin
     def refresh_watchers
       target_paths_hash = expand_paths
       existence_paths_hash = existence_path
-      
+
       log.debug {
         target_paths_str = target_paths_hash.collect { |key, target_info| target_info.path }.join(",")
         existence_paths_str = existence_paths_hash.collect { |key, target_info| target_info.path }.join(",")
         "tailing paths: target = #{target_paths_str} | existing = #{existence_paths_str}"
       }
 
-      unwatched_hash = existence_paths_hash.reject {|key, value| target_paths_hash.key?(key)}
+      if !@follow_inodes
+        need_unwatch_in_stop_watchers = true
+      else
+        # When using @follow_inodes, need this to unwatch the rotated old inode when it disappears.
+        # After `update_watcher` detaches an old TailWatcher, the inode is lost from the `@tails`.
+        # So that inode can't be contained in `removed_hash`, and can't be unwatched by `stop_watchers`.
+        #
+        # This logic may work for `@follow_inodes false` too.
+        # Just limiting the case to suppress the impact to existing logics.
+        @pf&.unwatch_removed_targets(target_paths_hash)
+        need_unwatch_in_stop_watchers = false
+      end
+
+      removed_hash = existence_paths_hash.reject {|key, value| target_paths_hash.key?(key)}
       added_hash = target_paths_hash.reject {|key, value| existence_paths_hash.key?(key)}
 
-      stop_watchers(unwatched_hash, immediate: false, unwatched: true) unless unwatched_hash.empty?
+      # If an exisiting TailWatcher already follows a target path with the different inode,
+      # it means that the TailWatcher following the rotated file still exists. In this case,
+      # `refresh_watcher` can't start the new TailWatcher for the new current file. So, we
+      # should output a warning log in order to prevent silent collection stops.
+      # (Such as https://github.com/fluent/fluentd/pull/4327)
+      # (Usually, such a TailWatcher should be removed from `@tails` in `update_watcher`.)
+      # (The similar warning may work for `@follow_inodes true` too. Just limiting the case
+      # to suppress the impact to existing logics.)
+      unless @follow_inodes
+        target_paths_hash.each do |path, target|
+          next unless @tails.key?(path)
+          # We can't use `existence_paths_hash[path].ino` because it is from `TailWatcher.ino`,
+          # which is very unstable parameter. (It can be `nil` or old).
+          # So, we need to use `TailWatcher.pe.read_inode`.
+          existing_watcher_inode = @tails[path].pe.read_inode
+          if existing_watcher_inode != target.ino
+            log.warn "Could not follow a file (inode: #{target.ino}) because an existing watcher for that filepath follows a different inode: #{existing_watcher_inode} (e.g. keeps watching a already rotated file). If you keep getting this message, please restart Fluentd.",
+              filepath: target.path
+          end
+        end
+      end
+
+      stop_watchers(removed_hash, unwatched: need_unwatch_in_stop_watchers) unless removed_hash.empty?
       start_watchers(added_hash) unless added_hash.empty?
       @startup = false if @startup
     end
@@ -484,8 +519,26 @@ module Fluent::Plugin
     end
 
     # refresh_watchers calls @tails.keys so we don't use stop_watcher -> start_watcher sequence for safety.
-    def update_watcher(target_info, pe)
-      path = target_info.path
+    def update_watcher(tail_watcher, pe, new_inode)
+      # TODO we should use another callback for this.
+      # To supress impact to existing logics, limit the case to `@follow_inodes`.
+      # We may not need `@follow_inodes` condition.
+      if @follow_inodes && new_inode.nil?
+        # nil inode means the file disappeared, so we only need to stop it.
+        @tails.delete(tail_watcher.path)
+        # https://github.com/fluent/fluentd/pull/4237#issuecomment-1633358632 
+        # Because of this problem, log duplication can occur during `rotate_wait`.
+        # Need to set `rotate_wait 0` for a workaround.
+        # Duplication will occur if `refresh_watcher` is called during the `rotate_wait`.
+        # In that case, `refresh_watcher` will add the new TailWatcher to tail the same target,
+        # and it causes the log duplication.
+        # (Other `detach_watcher_after_rotate_wait` may have the same problem.
+        #  We need the mechanism not to add duplicated TailWathcer with detaching TailWatcher.)
+        detach_watcher_after_rotate_wait(tail_watcher, pe.read_inode)
+        return
+      end
+
+      path = tail_watcher.path
 
       log.info("detected rotation of #{path}; waiting #{@rotate_wait} seconds")
 
@@ -499,23 +552,22 @@ module Fluent::Plugin
         end
       end
 
-      rotated_tw = @tails[path]
+      new_target_info = TargetInfo.new(path, new_inode)
 
       if @follow_inodes
-        new_position_entry = @pf[target_info]
-
+        new_position_entry = @pf[new_target_info]
+        # If `refresh_watcher` find the new file before, this will not be zero.
+        # In this case, only we have to do is detaching the current tail_watcher.
         if new_position_entry.read_inode == 0
-          # When follow_inodes is true, it's not cleaned up by refresh_watcher.
-          # So it should be unwatched here explicitly.
-          rotated_tw.unwatched = true if rotated_tw
-          @tails[path] = setup_watcher(target_info, new_position_entry)
+          @tails[path] = setup_watcher(new_target_info, new_position_entry)
           @tails[path].on_notify
         end
       else
-        @tails[path] = setup_watcher(target_info, pe)
+        @tails[path] = setup_watcher(new_target_info, pe)
         @tails[path].on_notify
       end
-      detach_watcher_after_rotate_wait(rotated_tw, pe.read_inode) if rotated_tw
+
+      detach_watcher_after_rotate_wait(tail_watcher, pe.read_inode)
     end
 
     # TailWatcher#close is called by another thread at shutdown phase.
@@ -523,6 +575,10 @@ module Fluent::Plugin
     # so adding close_io argument to avoid this problem.
     # At shutdown, IOHandler's io will be released automatically after detached the event loop
     def detach_watcher(tw, ino, close_io = true)
+      if @follow_inodes && tw.ino != ino
+        log.warn("detach_watcher could be detaching an unexpected tail_watcher with a different ino.",
+                  path: tw.path, actual_ino_in_tw: tw.ino, expect_ino_to_close: ino)
+      end
       tw.watchers.each do |watcher|
         event_loop_detach(watcher)
       end
@@ -530,7 +586,7 @@ module Fluent::Plugin
 
       tw.close if close_io
 
-      if tw.unwatched && @pf
+      if @pf && tw.unwatched && (@follow_inode || !@tails[tw.path])
         target_info = TargetInfo.new(tw.path, ino)
         @pf.unwatch(target_info)
       end
@@ -778,7 +834,7 @@ module Fluent::Plugin
       attr_accessor :group_watcher
 
       def tag
-        @parsed_tag ||= @path.tr('/', '.').gsub(/\.+/, '.').gsub(/^\./, '')
+        @parsed_tag ||= @path.tr('/', '.').squeeze('.').gsub(/^\./, '')
       end
 
       def register_watcher(watcher)
@@ -874,21 +930,21 @@ module Fluent::Plugin
 
           if watcher_needs_update
             if @follow_inodes
-              # No need to update a watcher if stat is nil (file not present), because moving to inodes will create
-              # new watcher, and old watcher will be closed by stop_watcher in refresh_watchers method
-              # don't want to swap state because we need latest read offset in pos file even after rotate_wait
-              if stat
-                target_info = TargetInfo.new(@path, stat.ino)
-                @update_watcher.call(target_info, @pe)
-              end
+              # If stat is nil (file not present), NEED to stop and discard this watcher.
+              #   When the file is disappeared but is resurrected soon, then `#refresh_watcher`
+              #   can't recognize this TailWatcher needs to be stopped.
+              #   This can happens when the file is rotated.
+              #   If a notify comes before the new file for the path is created during rotation,
+              #   then it appears as if the file was resurrected once it disappeared.
+              # Don't want to swap state because we need latest read offset in pos file even after rotate_wait
+              @update_watcher.call(self, @pe, stat&.ino)
             else
               # Permit to handle if stat is nil (file not present).
               # If a file is mv-ed and a new file is created during
               # calling `#refresh_watchers`s, and `#refresh_watchers` won't run `#start_watchers`
               # and `#stop_watchers()` for the path because `target_paths_hash`
               # always contains the path.
-              target_info = TargetInfo.new(@path, stat ? stat.ino : nil)
-              @update_watcher.call(target_info, swap_state(@pe))
+              @update_watcher.call(self, swap_state(@pe), stat&.ino)
             end
           else
             @log.info "detected rotation of #{@path}"

data/lib/fluent/plugin/out_exec_filter.rb

@@ -163,7 +163,7 @@ module Fluent::Plugin
                     0
                   elsif (@child_respawn == 'inf') || (@child_respawn == '-1')
                     -1
-                  elsif @child_respawn =~ /^\d+$/
+                  elsif /^\d+$/.match?(@child_respawn)
                     @child_respawn.to_i
                   else
                     raise ConfigError, "child_respawn option argument invalid: none(or 0), inf(or -1) or positive number"
@@ -187,7 +187,7 @@ module Fluent::Plugin
       @rr = 0
 
       exit_callback = ->(status){
-        c = @children.select{|child| child.pid == status.pid }.first
+        c = @children.find{|child| child.pid == status.pid }
         if c
           unless self.stopped?
             log.warn "child process exits with error code", code: status.to_i, status: status.exitstatus, signal: status.termsig

data/lib/fluent/plugin/output.rb

@@ -824,7 +824,7 @@ module Fluent
             if str.include?('${tag}')
               rvalue = rvalue.gsub('${tag}', metadata.tag)
             end
-            if str =~ CHUNK_TAG_PLACEHOLDER_PATTERN
+            if CHUNK_TAG_PLACEHOLDER_PATTERN.match?(str)
               hash = {}
               tag_parts = metadata.tag.split('.')
               tag_parts.each_with_index do |part, i|

data/lib/fluent/plugin/parser_json.rb

@@ -60,7 +60,7 @@ module Fluent
       rescue LoadError => ex
         name = :yajl
         if log
-          if /\boj\z/ =~ ex.message
+          if /\boj\z/.match?(ex.message)
             log.info "Oj is not installed, and failing back to Yajl for json parser"
           else
             log.warn ex.message

data/lib/fluent/plugin_helper/event_loop.rb

@@ -99,7 +99,7 @@ module Fluent
 
       def shutdown
         @_event_loop_mutex.synchronize do
-          @_event_loop_attached_watchers.reverse.each do |w|
+          @_event_loop_attached_watchers.reverse_each do |w|
             if w.attached?
               begin
                 w.detach
@@ -116,7 +116,7 @@ module Fluent
       def after_shutdown
         timeout_at = Fluent::Clock.now + EVENT_LOOP_SHUTDOWN_TIMEOUT
         @_event_loop_mutex.synchronize do
-          @_event_loop.watchers.reverse.each do |w|
+          @_event_loop.watchers.reverse_each do |w|
             begin
               w.detach
             rescue => e

data/lib/fluent/plugin_helper/record_accessor.rb

@@ -119,7 +119,7 @@ module Fluent
         def self.validate_dot_keys(keys)
           keys.each { |key|
             next unless key.is_a?(String)
-            if /\s+/.match(key)
+            if /\s+/.match?(key)
               raise Fluent::ConfigError, "whitespace character is not allowed in dot notation. Use bracket notation: #{key}"
             end
           }

data/lib/fluent/plugin_helper/thread.rb

@@ -101,16 +101,16 @@ module Fluent
       end
 
       def thread_exist?(title)
-        @_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.size > 0
+        @_threads.values.count{|thread| title == thread[:_fluentd_plugin_helper_thread_title] } > 0
       end
 
       def thread_started?(title)
-        t = @_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.first
+        t = @_threads.values.find{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }
         t && t[:_fluentd_plugin_helper_thread_started]
       end
 
       def thread_running?(title)
-        t = @_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.first
+        t = @_threads.values.find{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }
         t && t[:_fluentd_plugin_helper_thread_running]
       end
 

data/lib/fluent/plugin_id.rb

@@ -49,7 +49,7 @@ module Fluent
         # Thread::Backtrace::Location#path returns base filename or absolute path.
         # #absolute_path returns absolute_path always.
         # https://bugs.ruby-lang.org/issues/12159
-        if location.absolute_path =~ /\/test_[^\/]+\.rb$/ # location.path =~ /test_.+\.rb$/
+        if /\/test_[^\/]+\.rb$/.match?(location.absolute_path) # location.path =~ /test_.+\.rb$/
           return true
         end
       end

data/lib/fluent/supervisor.rb

@@ -697,7 +697,7 @@ module Fluent
       actual_log_path = @log_path
 
       # We need to prepare a unique path for each worker since Windows locks files.
-      if Fluent.windows? && rotate
+      if Fluent.windows? && rotate && @log_path && @log_path != "-"
         actual_log_path = Fluent::Log.per_process_path(@log_path, process_type, worker_id)
       end
 

data/lib/fluent/system_config.rb

@@ -62,7 +62,7 @@ module Fluent
       config_param :time_format, :string, default: '%Y-%m-%d %H:%M:%S %z'
       config_param :rotate_age, default: nil do |v|
         if Fluent::Log::LOG_ROTATE_AGE.include?(v)
-          v.to_sym
+          v
         else
           begin
             Integer(v)

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.16.1'
+  VERSION = '1.16.3'
 
 end

data/test/config/test_system_config.rb

@@ -151,7 +151,7 @@ module Fluent::Config
       data('daily' => "daily",
            'weekly' => 'weekly',
            'monthly' => 'monthly')
-      test "symbols for rotate_age" do |age|
+      test "strings for rotate_age" do |age|
         conf = parse_text(<<-EOS)
           <system>
             <log>
@@ -160,7 +160,7 @@ module Fluent::Config
           </system>
         EOS
         sc = Fluent::SystemConfig.new(conf)
-        assert_equal(age.to_sym, sc.log.rotate_age)
+        assert_equal(age, sc.log.rotate_age)
       end
 
       test "numeric number for rotate age" do

data/test/plugin/in_tail/test_position_file.rb

@@ -26,6 +26,10 @@ class IntailPositionFileTest < Test::Unit::TestCase
     "valid_path" => Fluent::Plugin::TailInput::TargetInfo.new("valid_path", 1),
     "inode23bit" => Fluent::Plugin::TailInput::TargetInfo.new("inode23bit", 0),
   }
+  TEST_CONTENT_INODES = {
+    1 => Fluent::Plugin::TailInput::TargetInfo.new("valid_path", 1),
+    0 => Fluent::Plugin::TailInput::TargetInfo.new("inode23bit", 0),
+  }
 
   def write_data(f, content)
     f.write(content)
@@ -221,7 +225,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
   end
 
   sub_test_case '#unwatch' do
-    test 'deletes entry by path' do
+    test 'unwatch entry by path' do
       write_data(@file, TEST_CONTENT)
       pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, logger: $log)
       inode1 = File.stat(@file).ino
@@ -239,6 +243,32 @@ class IntailPositionFileTest < Test::Unit::TestCase
 
       assert_not_equal p1, p2
     end
+
+    test 'unwatch entries by inode' do
+      write_data(@file, TEST_CONTENT)
+      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, true, TEST_CONTENT_INODES, logger: $log)
+
+      existing_targets = TEST_CONTENT_INODES.select do |inode, target_info|
+        inode == 1
+      end
+      pe_to_unwatch = pf[TEST_CONTENT_INODES[0]]
+
+      pf.unwatch_removed_targets(existing_targets)
+
+      assert_equal(
+        {
+          map_keys: [TEST_CONTENT_INODES[1].ino],
+          unwatched_pe_pos: Fluent::Plugin::TailInput::PositionFile::UNWATCHED_POSITION,
+        },
+        {
+          map_keys: pf.instance_variable_get(:@map).keys,
+          unwatched_pe_pos: pe_to_unwatch.read_pos,
+        }
+      )
+
+      unwatched_pe_retaken = pf[TEST_CONTENT_INODES[0]]
+      assert_not_equal pe_to_unwatch, unwatched_pe_retaken
+    end
   end
 
   sub_test_case 'FilePositionEntry' do

data/test/plugin/test_base.rb

@@ -108,7 +108,7 @@ class BaseTest < Test::Unit::TestCase
     @p.extend m
     assert_equal [], logger.logs
 
-    ret = @p.string_safe_encoding("abc\xff.\x01f"){|s| s.split(/\./) }
+    ret = @p.string_safe_encoding("abc\xff.\x01f"){|s| s.split(".") }
     assert_equal ['abc?', "\u0001f"], ret
     assert_equal 1, logger.logs.size
     assert{ logger.logs.first.include?("invalid byte sequence is replaced in ") }

data/test/plugin/test_buffer_chunk.rb

@@ -57,6 +57,17 @@ class BufferChunkTest < Test::Unit::TestCase
       assert chunk.respond_to?(:msgpack_each)
     end
 
+    test 'unpacker arg is not implemented for ChunkMessagePackEventStreamer' do
+      meta = Object.new
+      chunk = Fluent::Plugin::Buffer::Chunk.new(meta)
+      chunk.extend Fluent::ChunkMessagePackEventStreamer
+
+      unpacker = Fluent::MessagePackFactory.thread_local_msgpack_unpacker
+
+      assert_raise(NotImplementedError){ chunk.each(unpacker: unpacker) }
+      assert_raise(NotImplementedError){ chunk.msgpack_each(unpacker: unpacker) }
+    end
+
     test 'some methods raise ArgumentError with an option of `compressed: :gzip` and without extending Compressble`' do
       meta = Object.new
       chunk = Fluent::Plugin::Buffer::Chunk.new(meta)

data/test/plugin/test_in_forward.rb

@@ -367,7 +367,7 @@ class ForwardInputTest < Test::Unit::TestCase
       end
 
       logs = d.instance.log.out.logs
-      assert{ logs.select{|line| line =~ /skip invalid event/ }.size == 2 }
+      assert{ logs.count{|line| line =~ /skip invalid event/ } == 2 }
 
       d.instance_shutdown
     end
@@ -593,10 +593,10 @@ class ForwardInputTest < Test::Unit::TestCase
 
       # check log
       logs = d.instance.log.logs
-      assert_equal 1, logs.select{|line|
+      assert_equal 1, logs.count{|line|
         line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_warn_limit':/ &&
         line =~ / tag="test.tag" host="#{LOCALHOST_HOSTNAME}" limit=16777216 size=16777501/
-      }.size, "large chunk warning is not logged"
+      }, "large chunk warning is not logged"
 
       d.instance_shutdown
     end
@@ -619,10 +619,10 @@ class ForwardInputTest < Test::Unit::TestCase
 
       # check log
       logs = d.instance.log.logs
-      assert_equal 1, logs.select{ |line|
+      assert_equal 1, logs.count{ |line|
         line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_warn_limit':/ &&
         line =~ / tag="test.tag" host="#{LOCALHOST_HOSTNAME}" limit=16777216 size=16777501/
-      }.size, "large chunk warning is not logged"
+      }, "large chunk warning is not logged"
 
       d.instance_shutdown
     end
@@ -653,10 +653,10 @@ class ForwardInputTest < Test::Unit::TestCase
 
       # check log
       logs = d.instance.log.logs
-      assert_equal 1, logs.select{|line|
+      assert_equal 1, logs.count{|line|
         line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_limit', dropped:/ &&
         line =~ / tag="test.tag" host="#{LOCALHOST_HOSTNAME}" limit=33554432 size=33554989/
-      }.size, "large chunk warning is not logged"
+      }, "large chunk warning is not logged"
 
       d.instance_shutdown
     end
@@ -676,9 +676,9 @@ class ForwardInputTest < Test::Unit::TestCase
 
       # check log
       logs = d.instance.log.logs
-      assert_equal 1, logs.select{|line|
+      assert_equal 1, logs.count{|line|
         line =~ / \[warn\]: incoming chunk is broken: host="#{LOCALHOST_HOSTNAME}" msg=#{data.inspect}/
-      }.size, "should not accept broken chunk"
+      }, "should not accept broken chunk"
 
       d.instance_shutdown
     end