fluentd 1.16.2-x64-mingw32 → 1.16.3-x64-mingw32

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e245010ad651794b0c9a6324012373b8d7ef737d29fd91541977d3b9ccf28d03
-  data.tar.gz: af1d3135da23f2befd2da6e119474773e2bce8bc77bc7845509e4d29738f5db6
+  metadata.gz: c8ea6d9f4ae24382735ba31c0e49779cb12a0345205628e517b66e06f482a327
+  data.tar.gz: '059a1bd2d7f52ee127ed4f8f3a0c0ec439c83a55738d8ccc85ee0efade7f97ff'
 SHA512:
-  metadata.gz: 327c2f56c24b1eb80923a1b2864f52edd735967efb0406843a1b4d120330b8aa36cf36332d6e2174a4abbcf149fd8fd7c25230fb9171eeccd0de9df31ab0f4c7
-  data.tar.gz: 225659babf6d4e8d9e3d0a6ff809a7d5cc34a5fc7c129be3d03321017bdb5bb55602bcad2bfe86fed67cd311cdb15669b85529acb2aeedd4b0590cb7352eb4ab
+  metadata.gz: 16e4d6c562e263068bbf449fc7aeb6bb17c643a942fe2b78a96cdbd9d54b5ad7df912df31f43dda4c375569d7b9285e646acb033d478f16e576d798760aed834
+  data.tar.gz: a6a607ef479fefa25eb4b2b68a6d664a797807c957a111232bd926fe03c578b2cc7a1eac5c9db2304df81e43b44c6029c348cf70699ef32cdae8b12c6d4c6f1f

data/.github/workflows/linux-test.yaml

@@ -2,9 +2,9 @@ name: Testing on Ubuntu
 
 on:
   push:
-    branches: [master]
+    branches: [master, v1.16]
   pull_request:
-    branches: [master]
+    branches: [master, v1.16]
 
 jobs:
   test:

data/.github/workflows/macos-test.yaml

@@ -2,9 +2,9 @@ name: Testing on macOS
 
 on:
   push:
-    branches: [master]
+    branches: [master, v1.16]
   pull_request:
-    branches: [master]
+    branches: [master, v1.16]
 
 jobs:
   test:

data/.github/workflows/windows-test.yaml

@@ -2,9 +2,9 @@ name: Testing on Windows
 
 on:
   push:
-    branches: [master]
+    branches: [master, v1.16]
   pull_request:
-    branches: [master]
+    branches: [master, v1.16]
 
 jobs:
   test:

data/CHANGELOG.md

@@ -1,5 +1,18 @@
 # v1.16
 
+## Release v1.16.3 - 2023/11/14
+
+### Bug Fix
+
+* in_tail: Fix a stall bug on !follow_inode case
+  https://github.com/fluent/fluentd/pull/4327
+* in_tail: add warning for silent stop on !follow_inodes case
+  https://github.com/fluent/fluentd/pull/4339
+* Buffer: Fix NoMethodError with empty unstaged chunk arrays
+  https://github.com/fluent/fluentd/pull/4303
+* Fix for rotate_age where Fluentd passes as Symbol
+  https://github.com/fluent/fluentd/pull/4311
+
 ## Release v1.16.2 - 2023/07/14
 
 ### Bug Fix

data/Rakefile

@@ -13,7 +13,7 @@ task test: [:base_test]
 namespace :build do
   desc 'Build gems for all platforms'
   task :all do
-    Bundler.with_clean_env do
+    Bundler.with_original_env do
       %w[ruby x86-mingw32 x64-mingw32 x64-mingw-ucrt].each do |name|
         ENV['GEM_BUILD_FAKE_PLATFORM'] = name
         Rake::Task["build"].execute

data/lib/fluent/plugin/buffer.rb

@@ -417,7 +417,7 @@ module Fluent
             if c.staged? && (enqueue || chunk_size_full?(c))
               m = c.metadata
               enqueue_chunk(m)
-              if unstaged_chunks[m]
+              if unstaged_chunks[m] && !unstaged_chunks[m].empty?
                 u = unstaged_chunks[m].pop
                 u.synchronize do
                   if u.unstaged? && !chunk_size_full?(u)

data/lib/fluent/plugin/in_tail.rb

@@ -385,7 +385,7 @@ module Fluent::Plugin
         # So that inode can't be contained in `removed_hash`, and can't be unwatched by `stop_watchers`.
         #
         # This logic may work for `@follow_inodes false` too.
-        # Just limiting the case to supress the impact to existing logics.
+        # Just limiting the case to suppress the impact to existing logics.
         @pf&.unwatch_removed_targets(target_paths_hash)
         need_unwatch_in_stop_watchers = false
       end
@@ -393,6 +393,28 @@ module Fluent::Plugin
       removed_hash = existence_paths_hash.reject {|key, value| target_paths_hash.key?(key)}
       added_hash = target_paths_hash.reject {|key, value| existence_paths_hash.key?(key)}
 
+      # If an exisiting TailWatcher already follows a target path with the different inode,
+      # it means that the TailWatcher following the rotated file still exists. In this case,
+      # `refresh_watcher` can't start the new TailWatcher for the new current file. So, we
+      # should output a warning log in order to prevent silent collection stops.
+      # (Such as https://github.com/fluent/fluentd/pull/4327)
+      # (Usually, such a TailWatcher should be removed from `@tails` in `update_watcher`.)
+      # (The similar warning may work for `@follow_inodes true` too. Just limiting the case
+      # to suppress the impact to existing logics.)
+      unless @follow_inodes
+        target_paths_hash.each do |path, target|
+          next unless @tails.key?(path)
+          # We can't use `existence_paths_hash[path].ino` because it is from `TailWatcher.ino`,
+          # which is very unstable parameter. (It can be `nil` or old).
+          # So, we need to use `TailWatcher.pe.read_inode`.
+          existing_watcher_inode = @tails[path].pe.read_inode
+          if existing_watcher_inode != target.ino
+            log.warn "Could not follow a file (inode: #{target.ino}) because an existing watcher for that filepath follows a different inode: #{existing_watcher_inode} (e.g. keeps watching a already rotated file). If you keep getting this message, please restart Fluentd.",
+              filepath: target.path
+          end
+        end
+      end
+
       stop_watchers(removed_hash, unwatched: need_unwatch_in_stop_watchers) unless removed_hash.empty?
       start_watchers(added_hash) unless added_hash.empty?
       @startup = false if @startup
@@ -564,7 +586,7 @@ module Fluent::Plugin
 
       tw.close if close_io
 
-      if tw.unwatched && @pf
+      if @pf && tw.unwatched && (@follow_inode || !@tails[tw.path])
         target_info = TargetInfo.new(tw.path, ino)
         @pf.unwatch(target_info)
       end

data/lib/fluent/system_config.rb

@@ -62,7 +62,7 @@ module Fluent
       config_param :time_format, :string, default: '%Y-%m-%d %H:%M:%S %z'
       config_param :rotate_age, default: nil do |v|
         if Fluent::Log::LOG_ROTATE_AGE.include?(v)
-          v.to_sym
+          v
         else
           begin
             Integer(v)

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.16.2'
+  VERSION = '1.16.3'
 
 end

data/test/config/test_system_config.rb

@@ -151,7 +151,7 @@ module Fluent::Config
       data('daily' => "daily",
            'weekly' => 'weekly',
            'monthly' => 'monthly')
-      test "symbols for rotate_age" do |age|
+      test "strings for rotate_age" do |age|
         conf = parse_text(<<-EOS)
           <system>
             <log>
@@ -160,7 +160,7 @@ module Fluent::Config
           </system>
         EOS
         sc = Fluent::SystemConfig.new(conf)
-        assert_equal(age.to_sym, sc.log.rotate_age)
+        assert_equal(age, sc.log.rotate_age)
       end
 
       test "numeric number for rotate age" do

data/test/plugin/test_in_tail.rb

@@ -3017,4 +3017,109 @@ class TailInputTest < Test::Unit::TestCase
       )
     end
   end
+
+  sub_test_case "Update watchers for rotation without follow_inodes" do
+    # The scenario where in_tail wrongly unwatches the PositionEntry.
+    # This is reported in https://github.com/fluent/fluentd/issues/3614.
+    def test_refreshTW_during_rotation
+      config = config_element(
+        "ROOT",
+        "",
+        {
+          "path" => "#{@tmp_dir}/tail.txt0",
+          "pos_file" => "#{@tmp_dir}/tail.pos",
+          "tag" => "t1",
+          "format" => "none",
+          "read_from_head" => "true",
+          # In order to detach the old watcher quickly.
+          "rotate_wait" => "3s",
+          # In order to reproduce the same condition stably, ensure that `refresh_watchers` is not
+          # called by a timer.
+          "refresh_interval" => "1h",
+          # stat_watcher often calls `TailWatcher::on_notify` faster than creating a new log file,
+          # so disable it in order to reproduce the same condition stably.
+          "enable_stat_watcher" => "false",
+        }
+      )
+      d = create_driver(config, false)
+
+      tail_watchers = []
+      stub.proxy(d.instance).setup_watcher do |tw|
+        tail_watchers.append(tw)
+        tw
+      end
+
+      Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file1 log1"}
+
+      d.run(expect_records: 6, timeout: 15) do
+        Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file1 log2"}
+        FileUtils.move("#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt" + "1")
+
+        # This reproduces the following situation:
+        #     `refresh_watchers` is called during the rotation process and it detects the current file being lost.
+        #     Then it stops and unwatches the TailWatcher.
+        d.instance.refresh_watchers
+
+        Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file2 log1"}
+
+        # `watch_timer` calls `TailWatcher::on_notify`, and then `update_watcher` trys to add the new TailWatcher.
+        # After `rotate_wait` interval, the PositionEntry is unwatched.
+        # HOWEVER, the new TailWatcher is still using that PositionEntry, so this breaks the PositionFile!!
+        # That PositionEntry is removed from `PositionFile::map`, but it is still working and remaining in the real pos file.
+        sleep 5
+
+        # Append to the new current log file.
+        # The PositionEntry is updated although it does not exist in `PositionFile::map`.
+        #     `PositionFile::map`: empty
+        #     Real pos file: `.../tail.txt 0000000000000016 (inode)`
+        Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file2 log2"}
+
+        # Rotate again
+        [1, 0].each do |i|
+          FileUtils.move("#{@tmp_dir}/tail.txt#{i}", "#{@tmp_dir}/tail.txt#{i + 1}")
+        end
+        Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file3 log1"}
+
+        # `watch_timer` calls `TailWatcher::on_notify`, and then `update_watcher` trys to update the TailWatcher.
+        sleep 3
+
+        Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file3 log2"}
+
+        # Wait `rotate_wait` for file2 to make sure to close all IO handlers
+        sleep 3
+      end
+
+      inode_0 = tail_watchers[0]&.ino
+      inode_1 = tail_watchers[1]&.ino
+      inode_2 = tail_watchers[2]&.ino
+      record_values = d.events.collect { |event| event[2]["message"] }.sort
+      position_entries = []
+      Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r") do |f|
+        f.readlines(chomp: true).each do |line|
+          values = line.split("\t")
+          position_entries.append([values[0], values[1], values[2].to_i(16)])
+        end
+      end
+
+      assert_equal(
+        {
+          record_values: ["file1 log1", "file1 log2", "file2 log1", "file2 log2", "file3 log1", "file3 log2"],
+          tail_watcher_paths: ["#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0"],
+          tail_watcher_inodes: [inode_0, inode_1, inode_2],
+          tail_watcher_io_handler_opened_statuses: [false, false, false],
+          position_entries: [
+            # The recorded path is old, but it is no problem. The path is not used when using follow_inodes.
+            ["#{@tmp_dir}/tail.txt0", "0000000000000016", inode_2],
+          ],
+        },
+        {
+          record_values: record_values,
+          tail_watcher_paths: tail_watchers.collect { |tw| tw.path },
+          tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },
+          tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },
+          position_entries: position_entries
+        },
+      )
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.16.2
+  version: 1.16.3
 platform: x64-mingw32
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-07-14 00:00:00.000000000 Z
+date: 2023-11-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -1014,7 +1014,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.5
+rubygems_version: 3.4.19
 signing_key: 
 specification_version: 4
 summary: Fluentd event collector