fluentd 1.16.2-x86-mingw32 → 1.16.4-x86-mingw32

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: efdb31d738e884c0dfa33e5562f6e19bac8d1d607ec1ee7edf21a2fa233c9fff
-  data.tar.gz: 454ec661fd99411f095273fdd7141fccb6cd9ac8736589777ea69334afa1c6e9
+  metadata.gz: 55d9dff2b2c0dc6754261babb59c10b05cb4990b04366a10b6c3ed531e939ef6
+  data.tar.gz: 3fe6adb800c7f359b4dae8ede6c38710549da0bccd6181cc27c95cfa4ec5f776
 SHA512:
-  metadata.gz: 267c3e9465f1107cd66b6e812692cf7861f3473ac74a86b2ebdc9c882ad9e1ceb4e6b7e7526b1788f7c9e68a590acba54f6f11e536d25b47554fc4c7d562758c
-  data.tar.gz: 3f84e786dd90af725554afb2abaa09ceafb04001ede6bccc975e6a61f42fe69494d551416d0d4e21fbc9a531fd144568af67ae2a27537582293089acfc60e720
+  metadata.gz: eed2be6c185fc49df73504535018a7eaf7c5142a5fcebbc08d6e56cba93edc94d1c9ed159456bc52a6925c7e929c8bef4a2f2ca03850798ff52d90054eae849b
+  data.tar.gz: 5c2d5c814e6eeffdbafd831f0fa04ab79dc0f4979c3e3fa35dae941d7cec35c800b735f6c2efafacf91ae265e65d7d1d171ca941d28c60309399f74e61aad554

data/.github/workflows/{linux-test.yaml → test.yml}

@@ -1,27 +1,22 @@
-name: Testing on Ubuntu
+name: Test
 
 on:
   push:
-    branches: [master]
+    branches: [v1.16]
   pull_request:
-    branches: [master]
+    branches: [v1.16]
 
 jobs:
   test:
     runs-on: ${{ matrix.os }}
-    continue-on-error: ${{ matrix.experimental }}
+    continue-on-error: false
     strategy:
       fail-fast: false
       matrix:
-        ruby-version: ['3.2', '3.1', '3.0', '2.7']
-        os: [ubuntu-latest]
-        experimental: [false]
-        include:
-          - ruby-version: head
-            os: ubuntu-latest
-            experimental: true
+        os: ['ubuntu-latest', 'macos-latest', 'windows-latest']
+        ruby-version: ['3.3', '3.2', '3.1', '3.0', '2.7']
 
-    name: Unit testing with Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
+    name: Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3
       - name: Set up Ruby
@@ -29,8 +24,9 @@ jobs:
         with:
           ruby-version: ${{ matrix.ruby-version }}
       - name: Install addons
+        if: ${{ matrix.os == 'ubuntu-latest' }}
         run: sudo apt-get install libgmp3-dev libcap-ng-dev
       - name: Install dependencies
         run: bundle install
       - name: Run tests
-        run: bundle exec rake test
+        run: bundle exec rake test TESTOPTS=-v

data/CHANGELOG.md

@@ -1,5 +1,34 @@
 # v1.16
 
+## Release v1.16.4 - 2024/03/14
+
+### Bug Fix
+
+* Fix to avoid processing discarded chunks in write_step_by_step.
+  It fixes not to raise pile of IOError when many `chunk
+  bytes limit exceeds` errors are occurred.
+  https://github.com/fluent/fluentd/pull/4342
+* in_tail: Fix tail watchers in `rotate_wait` state not being managed.
+  https://github.com/fluent/fluentd/pull/4334
+
+### Misc
+
+* buffer: Avoid unnecessary log processing. It will improve performance.
+  https://github.com/fluent/fluentd/pull/4331
+
+## Release v1.16.3 - 2023/11/14
+
+### Bug Fix
+
+* in_tail: Fix a stall bug on !follow_inode case
+  https://github.com/fluent/fluentd/pull/4327
+* in_tail: add warning for silent stop on !follow_inodes case
+  https://github.com/fluent/fluentd/pull/4339
+* Buffer: Fix NoMethodError with empty unstaged chunk arrays
+  https://github.com/fluent/fluentd/pull/4303
+* Fix for rotate_age where Fluentd passes as Symbol
+  https://github.com/fluent/fluentd/pull/4311
+
 ## Release v1.16.2 - 2023/07/14
 
 ### Bug Fix

data/lib/fluent/plugin/buffer.rb

@@ -417,7 +417,7 @@ module Fluent
             if c.staged? && (enqueue || chunk_size_full?(c))
               m = c.metadata
               enqueue_chunk(m)
-              if unstaged_chunks[m]
+              if unstaged_chunks[m] && !unstaged_chunks[m].empty?
                 u = unstaged_chunks[m].pop
                 u.synchronize do
                   if u.unstaged? && !chunk_size_full?(u)
@@ -580,7 +580,7 @@ module Fluent
           chunk = @dequeued.delete(chunk_id)
           return false unless chunk # already purged by other thread
           @queue.unshift(chunk)
-          log.trace "chunk taken back", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: chunk.metadata
+          log.on_trace { log.trace "chunk taken back", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: chunk.metadata }
           @queued_num[chunk.metadata] += 1 # BUG if nil
           @dequeued_num[chunk.metadata] -= 1
         end
@@ -610,7 +610,7 @@ module Fluent
             @queued_num.delete(metadata)
             @dequeued_num.delete(metadata)
           end
-          log.trace "chunk purged", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata
+          log.on_trace { log.trace "chunk purged", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata }
         end
 
         nil
@@ -728,7 +728,6 @@ module Fluent
 
       def write_step_by_step(metadata, data, format, splits_count, &block)
         splits = []
-        errors = []
         if splits_count > data.size
           splits_count = data.size
         end
@@ -749,16 +748,14 @@ module Fluent
         modified_chunks = []
         modified_metadata = metadata
         get_next_chunk = ->(){
-          c = if staged_chunk_used
-                # Staging new chunk here is bad idea:
-                # Recovering whole state including newly staged chunks is much harder than current implementation.
-                modified_metadata = modified_metadata.dup_next
-                generate_chunk(modified_metadata)
-              else
-                synchronize { @stage[modified_metadata] ||= generate_chunk(modified_metadata).staged! }
-              end
-          modified_chunks << c
-          c
+          if staged_chunk_used
+            # Staging new chunk here is bad idea:
+            # Recovering whole state including newly staged chunks is much harder than current implementation.
+            modified_metadata = modified_metadata.dup_next
+            generate_chunk(modified_metadata)
+          else
+            synchronize { @stage[modified_metadata] ||= generate_chunk(modified_metadata).staged! }
+          end
         }
 
         writing_splits_index = 0
@@ -766,6 +763,8 @@ module Fluent
 
         while writing_splits_index < splits.size
           chunk = get_next_chunk.call
+          errors = []
+          modified_chunks << {chunk: chunk, adding_bytesize: 0, errors: errors}
           chunk.synchronize do
             raise ShouldRetry unless chunk.writable?
             staged_chunk_used = true if chunk.staged?
@@ -851,15 +850,18 @@ module Fluent
               raise
             end
 
-            block.call(chunk, chunk.bytesize - original_bytesize, errors)
-            errors = []
+            modified_chunks.last[:adding_bytesize] = chunk.bytesize - original_bytesize
           end
         end
+        modified_chunks.each do |data|
+          block.call(data[:chunk], data[:adding_bytesize], data[:errors])
+        end
       rescue ShouldRetry
-        modified_chunks.each do |mc|
-          mc.rollback rescue nil
-          if mc.unstaged?
-            mc.purge rescue nil
+        modified_chunks.each do |data|
+          chunk = data[:chunk]
+          chunk.rollback rescue nil
+          if chunk.unstaged?
+            chunk.purge rescue nil
           end
         end
         enqueue_chunk(metadata) if enqueue_chunk_before_retry

data/lib/fluent/plugin/in_tail.rb

@@ -52,6 +52,7 @@ module Fluent::Plugin
       super
       @paths = []
       @tails = {}
+      @tails_rotate_wait = {}
       @pf_file = nil
       @pf = nil
       @ignore_list = []
@@ -267,6 +268,9 @@ module Fluent::Plugin
       @shutdown_start_time = Fluent::Clock.now
       # during shutdown phase, don't close io. It should be done in close after all threads are stopped. See close.
       stop_watchers(existence_path, immediate: true, remove_watcher: false)
+      @tails_rotate_wait.keys.each do |tw|
+        detach_watcher(tw, @tails_rotate_wait[tw][:ino], false)
+      end
       @pf_file.close if @pf_file
 
       super
@@ -275,6 +279,7 @@ module Fluent::Plugin
     def close
       super
       # close file handles after all threads stopped (in #close of thread plugin helper)
+      # It may be because we need to wait IOHanlder.ready_to_shutdown()
       close_watcher_handles
     end
 
@@ -385,7 +390,7 @@ module Fluent::Plugin
         # So that inode can't be contained in `removed_hash`, and can't be unwatched by `stop_watchers`.
         #
         # This logic may work for `@follow_inodes false` too.
-        # Just limiting the case to supress the impact to existing logics.
+        # Just limiting the case to suppress the impact to existing logics.
         @pf&.unwatch_removed_targets(target_paths_hash)
         need_unwatch_in_stop_watchers = false
       end
@@ -393,6 +398,28 @@ module Fluent::Plugin
       removed_hash = existence_paths_hash.reject {|key, value| target_paths_hash.key?(key)}
       added_hash = target_paths_hash.reject {|key, value| existence_paths_hash.key?(key)}
 
+      # If an exisiting TailWatcher already follows a target path with the different inode,
+      # it means that the TailWatcher following the rotated file still exists. In this case,
+      # `refresh_watcher` can't start the new TailWatcher for the new current file. So, we
+      # should output a warning log in order to prevent silent collection stops.
+      # (Such as https://github.com/fluent/fluentd/pull/4327)
+      # (Usually, such a TailWatcher should be removed from `@tails` in `update_watcher`.)
+      # (The similar warning may work for `@follow_inodes true` too. Just limiting the case
+      # to suppress the impact to existing logics.)
+      unless @follow_inodes
+        target_paths_hash.each do |path, target|
+          next unless @tails.key?(path)
+          # We can't use `existence_paths_hash[path].ino` because it is from `TailWatcher.ino`,
+          # which is very unstable parameter. (It can be `nil` or old).
+          # So, we need to use `TailWatcher.pe.read_inode`.
+          existing_watcher_inode = @tails[path].pe.read_inode
+          if existing_watcher_inode != target.ino
+            log.warn "Could not follow a file (inode: #{target.ino}) because an existing watcher for that filepath follows a different inode: #{existing_watcher_inode} (e.g. keeps watching a already rotated file). If you keep getting this message, please restart Fluentd.",
+              filepath: target.path
+          end
+        end
+      end
+
       stop_watchers(removed_hash, unwatched: need_unwatch_in_stop_watchers) unless removed_hash.empty?
       start_watchers(added_hash) unless added_hash.empty?
       @startup = false if @startup
@@ -494,6 +521,9 @@ module Fluent::Plugin
           tw.close
         end
       end
+      @tails_rotate_wait.keys.each do |tw|
+        tw.close
+      end
     end
 
     # refresh_watchers calls @tails.keys so we don't use stop_watcher -> start_watcher sequence for safety.
@@ -548,10 +578,6 @@ module Fluent::Plugin
       detach_watcher_after_rotate_wait(tail_watcher, pe.read_inode)
     end
 
-    # TailWatcher#close is called by another thread at shutdown phase.
-    # It causes 'can't modify string; temporarily locked' error in IOHandler
-    # so adding close_io argument to avoid this problem.
-    # At shutdown, IOHandler's io will be released automatically after detached the event loop
     def detach_watcher(tw, ino, close_io = true)
       if @follow_inodes && tw.ino != ino
         log.warn("detach_watcher could be detaching an unexpected tail_watcher with a different ino.",
@@ -564,7 +590,7 @@ module Fluent::Plugin
 
       tw.close if close_io
 
-      if tw.unwatched && @pf
+      if @pf && tw.unwatched && (@follow_inode || !@tails[tw.path])
         target_info = TargetInfo.new(tw.path, ino)
         @pf.unwatch(target_info)
       end
@@ -582,7 +608,11 @@ module Fluent::Plugin
       if @open_on_every_update
         # Detach now because it's already closed, waiting it doesn't make sense.
         detach_watcher(tw, ino)
-      elsif throttling_is_enabled?(tw)
+      end
+
+      return if @tails_rotate_wait[tw]
+
+      if throttling_is_enabled?(tw)
         # When the throttling feature is enabled, it might not reach EOF yet.
         # Should ensure to read all contents before closing it, with keeping throttling.
         start_time_to_wait = Fluent::Clock.now
@@ -590,14 +620,18 @@ module Fluent::Plugin
           elapsed = Fluent::Clock.now - start_time_to_wait
           if tw.eof? && elapsed >= @rotate_wait
             timer.detach
+            @tails_rotate_wait.delete(tw)
             detach_watcher(tw, ino)
           end
         end
+        @tails_rotate_wait[tw] = { ino: ino, timer: timer }
       else
         # when the throttling feature isn't enabled, just wait @rotate_wait
-        timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do
+        timer = timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do
+          @tails_rotate_wait.delete(tw)
           detach_watcher(tw, ino)
         end
+        @tails_rotate_wait[tw] = { ino: ino, timer: timer }
       end
     end
 

data/lib/fluent/system_config.rb

@@ -62,7 +62,7 @@ module Fluent
       config_param :time_format, :string, default: '%Y-%m-%d %H:%M:%S %z'
       config_param :rotate_age, default: nil do |v|
         if Fluent::Log::LOG_ROTATE_AGE.include?(v)
-          v.to_sym
+          v
         else
           begin
             Integer(v)

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.16.2'
+  VERSION = '1.16.4'
 
 end

data/test/command/test_fluentd.rb

@@ -941,7 +941,7 @@ CONF
       '-external-encoding' => '--external-encoding=utf-8',
       '-internal-encoding' => '--internal-encoding=utf-8',
     )
-    test "-E option is set to RUBYOPT" do |opt|
+    test "-E option is set to RUBYOPT" do |base_opt|
       conf = <<CONF
 <source>
   @type dummy
@@ -952,6 +952,7 @@ CONF
 </match>
 CONF
       conf_path = create_conf_file('rubyopt_test.conf', conf)
+      opt = base_opt.dup
       opt << " #{ENV['RUBYOPT']}" if ENV['RUBYOPT']
       assert_log_matches(
         create_cmdline(conf_path),
@@ -991,9 +992,14 @@ CONF
 </match>
 CONF
       conf_path = create_conf_file('rubyopt_invalid_test.conf', conf)
+      if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('3.3.0')
+        expected_phrase = 'ruby: invalid switch in RUBYOPT'
+      else
+        expected_phrase = 'Invalid option is passed to RUBYOPT'
+      end
       assert_log_matches(
         create_cmdline(conf_path),
-        'Invalid option is passed to RUBYOPT',
+        expected_phrase,
         env: { 'RUBYOPT' => 'a' },
       )
     end

data/test/config/test_system_config.rb

@@ -151,7 +151,7 @@ module Fluent::Config
       data('daily' => "daily",
            'weekly' => 'weekly',
            'monthly' => 'monthly')
-      test "symbols for rotate_age" do |age|
+      test "strings for rotate_age" do |age|
         conf = parse_text(<<-EOS)
           <system>
             <log>
@@ -160,7 +160,7 @@ module Fluent::Config
           </system>
         EOS
         sc = Fluent::SystemConfig.new(conf)
-        assert_equal(age.to_sym, sc.log.rotate_age)
+        assert_equal(age, sc.log.rotate_age)
       end
 
       test "numeric number for rotate age" do

data/test/plugin/test_buffer.rb

@@ -850,6 +850,57 @@ class BufferTest < Test::Unit::TestCase
     test '#compress returns :text' do
       assert_equal :text, @p.compress
     end
+
+    # https://github.com/fluent/fluentd/issues/3089
+    test "closed chunk should not be committed" do
+      assert_equal 8 * 1024 * 1024, @p.chunk_limit_size
+      assert_equal 0.95, @p.chunk_full_threshold
+
+      purge_count = 0
+
+      stub.proxy(@p).generate_chunk(anything) do |chunk|
+        stub.proxy(chunk).purge do |result|
+          purge_count += 1
+          result
+        end
+        stub.proxy(chunk).commit do |result|
+          assert_false(chunk.closed?)
+          result
+        end
+        stub.proxy(chunk).rollback do |result|
+          assert_false(chunk.closed?)
+          result
+        end
+        chunk
+      end
+
+      m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)
+      small_row = "x" * 1024 * 400
+      big_row = "x" * 1024 * 1024 * 8 # just `chunk_size_limit`, it does't cause BufferOverFlowError.
+
+      # Write 42 events in 1 event stream, last one is for triggering `ShouldRetry`
+      @p.write({m => [small_row] * 40 + [big_row] + ["x"]})
+
+      # Above event strem will be splitted twice by `Buffer#write_step_by_step`
+      #
+      # 1. `write_once`: 42 [events] * 1 [stream]
+      # 2. `write_step_by_step`: 4 [events]* 10 [streams] + 2 [events] * 1 [stream]
+      # 3. `write_step_by_step` (by `ShouldRetry`): 1 [event] * 42 [streams]
+      #
+      # The problematic data is built in the 2nd stage.
+      # In the 2nd stage, 5 streams are packed in a chunk.
+      # ((1024 * 400) [bytes] * 4 [events] * 5 [streams] = 8192000 [bytes] < `chunk_limit_size` (8MB)).
+      # So 3 chunks are used to store all data.
+      # The 1st chunk is already staged by `write_once`.
+      # The 2nd & 3rd chunks are newly created as unstaged.
+      # The 3rd chunk is purged before `ShouldRetry`, it's no problem:
+      #   https://github.com/fluent/fluentd/blob/7e9eba736ff40ad985341be800ddc46558be75f2/lib/fluent/plugin/buffer.rb#L850
+      # The 2nd chunk is purged in `rescue ShouldRetry`:
+      #   https://github.com/fluent/fluentd/blob/7e9eba736ff40ad985341be800ddc46558be75f2/lib/fluent/plugin/buffer.rb#L862
+      # It causes the issue described in https://github.com/fluent/fluentd/issues/3089#issuecomment-1811839198
+
+      assert_equal 2, purge_count
+    end
   end
 
   sub_test_case 'standard format with configuration for test with lower chunk limit size' do

data/test/plugin/test_in_tail.rb

@@ -3016,5 +3016,273 @@ class TailInputTest < Test::Unit::TestCase
         },
       )
     end
+
+    def test_next_rotation_occurs_very_fast_while_old_TW_still_waiting_rotate_wait
+      config = config_element(
+        "ROOT",
+        "",
+        {
+          "path" => "#{@tmp_dir}/tail.txt*",
+          "pos_file" => "#{@tmp_dir}/tail.pos",
+          "tag" => "t1",
+          "format" => "none",
+          "read_from_head" => "true",
+          "follow_inodes" => "true",
+          "rotate_wait" => "3s",
+          "refresh_interval" => "1h",
+          # stat_watcher often calls `TailWatcher::on_notify` faster than creating a new log file,
+          # so disable it in order to reproduce the same condition stably.
+          "enable_stat_watcher" => "false",
+        }
+      )
+      d = create_driver(config, false)
+
+      tail_watchers = []
+      stub.proxy(d.instance).setup_watcher do |tw|
+        tail_watchers.append(tw)
+        mock.proxy(tw).close.once # Note: Currently, there is no harm in duplicate calls.
+        tw
+      end
+
+      Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file1 log1"}
+
+      d.run(expect_records: 6, timeout: 15) do
+        Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file1 log2"}
+
+        sleep 1.5 # Need to be larger than 1s (the interval of watch_timer)
+
+        FileUtils.move("#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt" + "1")
+        Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file2 log1"}
+        Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file2 log2"}
+
+        sleep 1.5 # Need to be larger than 1s (the interval of watch_timer)
+
+        # Rotate again (Old TailWatcher waiting rotate_wait also calls update_watcher)
+        [1, 0].each do |i|
+          FileUtils.move("#{@tmp_dir}/tail.txt#{i}", "#{@tmp_dir}/tail.txt#{i + 1}")
+        end
+        Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file3 log1"}
+        Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file3 log2"}
+
+        # Wait rotate_wait to confirm that TailWatcher.close is not called in duplicate.
+        # (Note: Currently, there is no harm in duplicate calls)
+        sleep 4
+      end
+
+      inode_0 = tail_watchers[0]&.ino
+      inode_1 = tail_watchers[1]&.ino
+      inode_2 = tail_watchers[2]&.ino
+      record_values = d.events.collect { |event| event[2]["message"] }.sort
+      position_entries = []
+      Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r") do |f|
+        f.readlines(chomp: true).each do |line|
+          values = line.split("\t")
+          position_entries.append([values[0], values[1], values[2].to_i(16)])
+        end
+      end
+
+      assert_equal(
+        {
+          record_values: ["file1 log1", "file1 log2", "file2 log1", "file2 log2", "file3 log1", "file3 log2"],
+          tail_watcher_paths: ["#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0"],
+          tail_watcher_inodes: [inode_0, inode_1, inode_2],
+          tail_watcher_io_handler_opened_statuses: [false, false, false],
+          position_entries: [
+            ["#{@tmp_dir}/tail.txt0", "0000000000000016", inode_0],
+            ["#{@tmp_dir}/tail.txt0", "0000000000000016", inode_1],
+            ["#{@tmp_dir}/tail.txt0", "0000000000000016", inode_2],
+          ],
+        },
+        {
+          record_values: record_values,
+          tail_watcher_paths: tail_watchers.collect { |tw| tw.path },
+          tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },
+          tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },
+          position_entries: position_entries
+        },
+      )
+    end
+  end
+
+  sub_test_case "Update watchers for rotation without follow_inodes" do
+    # The scenario where in_tail wrongly unwatches the PositionEntry.
+    # This is reported in https://github.com/fluent/fluentd/issues/3614.
+    def test_refreshTW_during_rotation
+      config = config_element(
+        "ROOT",
+        "",
+        {
+          "path" => "#{@tmp_dir}/tail.txt0",
+          "pos_file" => "#{@tmp_dir}/tail.pos",
+          "tag" => "t1",
+          "format" => "none",
+          "read_from_head" => "true",
+          # In order to detach the old watcher quickly.
+          "rotate_wait" => "3s",
+          # In order to reproduce the same condition stably, ensure that `refresh_watchers` is not
+          # called by a timer.
+          "refresh_interval" => "1h",
+          # stat_watcher often calls `TailWatcher::on_notify` faster than creating a new log file,
+          # so disable it in order to reproduce the same condition stably.
+          "enable_stat_watcher" => "false",
+        }
+      )
+      d = create_driver(config, false)
+
+      tail_watchers = []
+      stub.proxy(d.instance).setup_watcher do |tw|
+        tail_watchers.append(tw)
+        tw
+      end
+
+      Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file1 log1"}
+
+      d.run(expect_records: 6, timeout: 15) do
+        Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file1 log2"}
+        FileUtils.move("#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt" + "1")
+
+        # This reproduces the following situation:
+        #     `refresh_watchers` is called during the rotation process and it detects the current file being lost.
+        #     Then it stops and unwatches the TailWatcher.
+        d.instance.refresh_watchers
+
+        Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file2 log1"}
+
+        # `watch_timer` calls `TailWatcher::on_notify`, and then `update_watcher` trys to add the new TailWatcher.
+        # After `rotate_wait` interval, the PositionEntry is unwatched.
+        # HOWEVER, the new TailWatcher is still using that PositionEntry, so this breaks the PositionFile!!
+        # That PositionEntry is removed from `PositionFile::map`, but it is still working and remaining in the real pos file.
+        sleep 5
+
+        # Append to the new current log file.
+        # The PositionEntry is updated although it does not exist in `PositionFile::map`.
+        #     `PositionFile::map`: empty
+        #     Real pos file: `.../tail.txt 0000000000000016 (inode)`
+        Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file2 log2"}
+
+        # Rotate again
+        [1, 0].each do |i|
+          FileUtils.move("#{@tmp_dir}/tail.txt#{i}", "#{@tmp_dir}/tail.txt#{i + 1}")
+        end
+        Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file3 log1"}
+
+        # `watch_timer` calls `TailWatcher::on_notify`, and then `update_watcher` trys to update the TailWatcher.
+        sleep 3
+
+        Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file3 log2"}
+      end
+
+      inode_0 = tail_watchers[0]&.ino
+      inode_1 = tail_watchers[1]&.ino
+      inode_2 = tail_watchers[2]&.ino
+      record_values = d.events.collect { |event| event[2]["message"] }.sort
+      position_entries = []
+      Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r") do |f|
+        f.readlines(chomp: true).each do |line|
+          values = line.split("\t")
+          position_entries.append([values[0], values[1], values[2].to_i(16)])
+        end
+      end
+
+      assert_equal(
+        {
+          record_values: ["file1 log1", "file1 log2", "file2 log1", "file2 log2", "file3 log1", "file3 log2"],
+          tail_watcher_paths: ["#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0"],
+          tail_watcher_inodes: [inode_0, inode_1, inode_2],
+          tail_watcher_io_handler_opened_statuses: [false, false, false],
+          position_entries: [
+            # The recorded path is old, but it is no problem. The path is not used when using follow_inodes.
+            ["#{@tmp_dir}/tail.txt0", "0000000000000016", inode_2],
+          ],
+        },
+        {
+          record_values: record_values,
+          tail_watcher_paths: tail_watchers.collect { |tw| tw.path },
+          tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },
+          tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },
+          position_entries: position_entries
+        },
+      )
+    end
+
+    def test_next_rotation_occurs_very_fast_while_old_TW_still_waiting_rotate_wait
+      config = config_element(
+        "ROOT",
+        "",
+        {
+          "path" => "#{@tmp_dir}/tail.txt0",
+          "pos_file" => "#{@tmp_dir}/tail.pos",
+          "tag" => "t1",
+          "format" => "none",
+          "read_from_head" => "true",
+          "rotate_wait" => "3s",
+          "refresh_interval" => "1h",
+        }
+      )
+      d = create_driver(config, false)
+
+      tail_watchers = []
+      stub.proxy(d.instance).setup_watcher do |tw|
+        tail_watchers.append(tw)
+        mock.proxy(tw).close.once # Note: Currently, there is no harm in duplicate calls.
+        tw
+      end
+
+      Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file1 log1"}
+
+      d.run(expect_records: 6, timeout: 15) do
+        Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file1 log2"}
+
+        sleep 1.5 # Need to be larger than 1s (the interval of watch_timer)
+
+        FileUtils.move("#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt" + "1")
+        Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file2 log1"}
+        Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file2 log2"}
+
+        sleep 1.5 # Need to be larger than 1s (the interval of watch_timer)
+
+        # Rotate again (Old TailWatcher waiting rotate_wait also calls update_watcher)
+        [1, 0].each do |i|
+          FileUtils.move("#{@tmp_dir}/tail.txt#{i}", "#{@tmp_dir}/tail.txt#{i + 1}")
+        end
+        Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file3 log1"}
+        Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file3 log2"}
+
+        # Wait rotate_wait to confirm that TailWatcher.close is not called in duplicate.
+        # (Note: Currently, there is no harm in duplicate calls)
+        sleep 4
+      end
+
+      inode_0 = tail_watchers[0]&.ino
+      inode_1 = tail_watchers[1]&.ino
+      inode_2 = tail_watchers[2]&.ino
+      record_values = d.events.collect { |event| event[2]["message"] }.sort
+      position_entries = []
+      Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r") do |f|
+        f.readlines(chomp: true).each do |line|
+          values = line.split("\t")
+          position_entries.append([values[0], values[1], values[2].to_i(16)])
+        end
+      end
+
+      assert_equal(
+        {
+          record_values: ["file1 log1", "file1 log2", "file2 log1", "file2 log2", "file3 log1", "file3 log2"],
+          tail_watcher_paths: ["#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0"],
+          tail_watcher_inodes: [inode_0, inode_1, inode_2],
+          tail_watcher_io_handler_opened_statuses: [false, false, false],
+          position_entries: [
+            ["#{@tmp_dir}/tail.txt0", "0000000000000016", inode_2],
+          ],
+        },
+        {
+          record_values: record_values,
+          tail_watcher_paths: tail_watchers.collect { |tw| tw.path },
+          tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },
+          tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },
+          position_entries: position_entries
+        },
+      )
+    end
   end
 end

data/test/plugin/test_out_forward.rb

@@ -156,7 +156,14 @@ EOL
     normal_conf = config_element('match', '**', {}, [
         config_element('server', '', {'name' => 'test', 'host' => 'unexisting.yaaaaaaaaaaaaaay.host.example.com'})
       ])
-    assert_raise SocketError do
+
+    if Socket.const_defined?(:ResolutionError) # as of Ruby 3.3
+      error_class = Socket::ResolutionError
+    else
+      error_class = SocketError
+    end
+
+    assert_raise error_class do
       create_driver(normal_conf)
     end
 
@@ -165,7 +172,7 @@ EOL
       ])
     @d = d = create_driver(conf)
     expected_log = "failed to resolve node name when configured"
-    expected_detail = 'server="test" error_class=SocketError'
+    expected_detail = "server=\"test\" error_class=#{error_class.name}"
     logs = d.logs
     assert{ logs.any?{|log| log.include?(expected_log) && log.include?(expected_detail) } }
   end
@@ -1241,27 +1248,22 @@ EOL
     target_input_driver = create_target_input_driver(conf: target_config)
     output_conf = config
     d = create_driver(output_conf)
-    d.instance_start
 
-    begin
-      chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
-      mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
-                                               linger_timeout: anything,
-                                               send_timeout: anything,
-                                               recv_timeout: anything,
-                                               connect_timeout: anything
-                                              ) { |sock| mock(sock).close.once; sock }.twice
+    chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
+    mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
+                                             linger_timeout: anything,
+                                             send_timeout: anything,
+                                             recv_timeout: anything,
+                                             connect_timeout: anything
+                                            ) { |sock| mock(sock).close.once; sock }.twice
 
-      target_input_driver.run(timeout: 15) do
-        d.run(shutdown: false) do
-          node = d.instance.nodes.first
-          2.times do
-            node.send_data('test', chunk) rescue nil
-          end
+    target_input_driver.run(timeout: 15) do
+      d.run do
+        node = d.instance.nodes.first
+        2.times do
+          node.send_data('test', chunk) rescue nil
         end
       end
-    ensure
-      d.instance_shutdown
     end
   end
 
@@ -1275,7 +1277,6 @@ EOL
       port #{@target_port}
     </server>
     ])
-    d.instance_start
     assert_nothing_raised { d.run }
   end
 
@@ -1287,33 +1288,28 @@ EOL
         keepalive_timeout 2
       ]
       d = create_driver(output_conf)
-      d.instance_start
 
-      begin
-        chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
-        mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
-                                                 linger_timeout: anything,
-                                                 send_timeout: anything,
-                                                 recv_timeout: anything,
-                                                 connect_timeout: anything
-                                                ) { |sock| mock(sock).close.once; sock }.once
+      chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
+      mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
+                                               linger_timeout: anything,
+                                               send_timeout: anything,
+                                               recv_timeout: anything,
+                                               connect_timeout: anything
+                                              ) { |sock| mock(sock).close.once; sock }.once
 
-        target_input_driver.run(timeout: 15) do
-          d.run(shutdown: false) do
-            node = d.instance.nodes.first
-            2.times do
-              node.send_data('test', chunk) rescue nil
-            end
+      target_input_driver.run(timeout: 15) do
+        d.run do
+          node = d.instance.nodes.first
+          2.times do
+            node.send_data('test', chunk) rescue nil
           end
         end
-      ensure
-        d.instance_shutdown
       end
     end
 
     test 'create timer of purging obsolete sockets' do
       output_conf = config + %[keepalive true]
-      d = create_driver(output_conf)
+      @d = d = create_driver(output_conf)
 
       mock(d.instance).timer_execute(:out_forward_heartbeat_request, 1).once
       mock(d.instance).timer_execute(:out_forward_keep_alived_socket_watcher, 5).once
@@ -1329,7 +1325,6 @@ EOL
           keepalive_timeout 2
         ]
         d = create_driver(output_conf)
-        d.instance_start
 
         chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
         mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,

data/test/plugin_helper/test_child_process.rb

@@ -515,6 +515,9 @@ class ChildProcessTest < Test::Unit::TestCase
   end
 
   test 'can scrub characters without exceptions' do
+    if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('3.3.0')
+      pend "Behaviour of IO#set_encoding is changed as of Ruby 3.3 (#4058)"
+    end
     m = Mutex.new
     str = nil
     Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
@@ -529,19 +532,25 @@ class ChildProcessTest < Test::Unit::TestCase
       sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
       m.lock
       assert_equal Encoding.find('utf-8'), str.encoding
-      expected = "\xEF\xBF\xBD\xEF\xBF\xBD\x00\xEF\xBF\xBD\xEF\xBF\xBD".force_encoding("utf-8")
+      replacement = "\uFFFD" # U+FFFD (REPLACEMENT CHARACTER)
+      nul = "\x00" # U+0000 (NUL)
+      expected = replacement * 2 + nul + replacement * 2
       assert_equal expected, str
       @d.stop; @d.shutdown; @d.close; @d.terminate
     end
   end
 
   test 'can scrub characters without exceptions and replace specified chars' do
+    if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('3.3.0')
+      pend "Behaviour of IO#set_encoding is changed as of Ruby 3.3 (#4058)"
+    end
     m = Mutex.new
     str = nil
+    replacement = "?"
     Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
       ran = false
       args = ['-e', 'STDOUT.set_encoding("ascii-8bit"); STDOUT.write "\xFF\xFF\x00\xF0\xF0"']
-      @d.child_process_execute(:t13b, "ruby", arguments: args, mode: [:read], scrub: true, replace_string: '?') do |io|
+      @d.child_process_execute(:t13b, "ruby", arguments: args, mode: [:read], scrub: true, replace_string: replacement) do |io|
         m.lock
         ran = true
         str = io.read
@@ -550,7 +559,8 @@ class ChildProcessTest < Test::Unit::TestCase
       sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
       m.lock
       assert_equal Encoding.find('utf-8'), str.encoding
-      expected = "??\x00??".force_encoding("utf-8")
+      nul = "\x00" # U+0000 (NUL)
+      expected = replacement * 2 + nul + replacement * 2
       assert_equal expected, str
       @d.stop; @d.shutdown; @d.close; @d.terminate
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.16.2
+  version: 1.16.4
 platform: x86-mingw32
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-07-14 00:00:00.000000000 Z
+date: 2024-03-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -436,14 +436,12 @@ extra_rdoc_files: []
 files:
 - ".deepsource.toml"
 - ".github/ISSUE_TEMPLATE.md"
-- ".github/ISSUE_TEMPLATE/bug_report.yaml"
+- ".github/ISSUE_TEMPLATE/bug_report.yml"
 - ".github/ISSUE_TEMPLATE/config.yml"
-- ".github/ISSUE_TEMPLATE/feature_request.yaml"
+- ".github/ISSUE_TEMPLATE/feature_request.yml"
 - ".github/PULL_REQUEST_TEMPLATE.md"
-- ".github/workflows/linux-test.yaml"
-- ".github/workflows/macos-test.yaml"
 - ".github/workflows/stale-actions.yml"
-- ".github/workflows/windows-test.yaml"
+- ".github/workflows/test.yml"
 - ".gitignore"
 - ADOPTERS.md
 - AUTHORS
@@ -1014,7 +1012,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.5
+rubygems_version: 3.4.19
 signing_key: 
 specification_version: 4
 summary: Fluentd event collector

data/.github/workflows/macos-test.yaml

@@ -1,34 +0,0 @@
-name: Testing on macOS
-
-on:
-  push:
-    branches: [master]
-  pull_request:
-    branches: [master]
-
-jobs:
-  test:
-    runs-on: ${{ matrix.os }}
-    continue-on-error: ${{ matrix.experimental }}
-    strategy:
-      fail-fast: false
-      matrix:
-        ruby-version: ['3.2', '3.1', '3.0', '2.7']
-        os: [macos-latest]
-        experimental: [true]
-        include:
-          - ruby-version: head
-            os: macos-latest
-            experimental: true
-
-    name: Unit testing with Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Ruby
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: ${{ matrix.ruby-version }}
-      - name: Install dependencies
-        run: bundle install
-      - name: Run tests
-        run: bundle exec rake test

data/.github/workflows/windows-test.yaml

@@ -1,49 +0,0 @@
-name: Testing on Windows
-
-on:
-  push:
-    branches: [master]
-  pull_request:
-    branches: [master]
-
-jobs:
-  test:
-    runs-on: ${{ matrix.os }}
-    continue-on-error: ${{ matrix.experimental }}
-    strategy:
-      fail-fast: false
-      matrix:
-        ruby-version: ['3.2', '3.1', '2.7']
-        os:
-          - windows-latest
-        experimental: [false]
-        include:
-          - ruby-version: head
-            os: windows-latest
-            experimental: true
-          - ruby-version: '3.0.3'
-            os: windows-latest
-            experimental: false
-            # On Ruby 3.0, we need to use fiddle 1.0.8 or later to retrieve correct
-            # error code. In addition, we have to specify the path of fiddle by RUBYLIB
-            # because RubyInstaller loads Ruby's bundled fiddle before initializing gem.
-            # See also:
-            # * https://github.com/ruby/fiddle/issues/72
-            # * https://bugs.ruby-lang.org/issues/17813
-            # * https://github.com/oneclick/rubyinstaller2/blob/8225034c22152d8195bc0aabc42a956c79d6c712/lib/ruby_installer/build/dll_directory.rb
-            ruby-lib-opt: RUBYLIB=%RUNNER_TOOL_CACHE%/Ruby/3.0.3/x64/lib/ruby/gems/3.0.0/gems/fiddle-1.1.0/lib
-
-    name: Unit testing with Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Ruby
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: ${{ matrix.ruby-version }}
-      - name: Add Fiddle 1.1.0
-        if: ${{ matrix.ruby-version == '3.0.3' }}
-        run: gem install fiddle --version 1.1.0
-      - name: Install dependencies
-        run: ridk exec bundle install
-      - name: Run tests
-        run: bundle exec rake test TESTOPTS=-v ${{ matrix.ruby-lib-opt }}