fluentd 1.15.2-x86-mingw32 → 1.15.3-x86-mingw32

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 151a6147f8d056032028f95746086afaa48f9d6335c41196a2269fb8c9ed1fdd
-  data.tar.gz: bc7da3f9b2bc71818d69dad63d5692365d09c75bab57b880c1900c2e7c47c404
+  metadata.gz: 126fe0c708fd1841d5b8d6eb273ccb15a88c68b255efb039a4b3669283e98db8
+  data.tar.gz: 9a90482923948c6b29f6628a1854db956e6838f54a281e4627497f8e9acd67c6
 SHA512:
-  metadata.gz: 8c396e5fb10f0ee63082445ab087b1dc556b5271d28a334ae5bad41ec8d570db4f85d03153ec43418f050dc23536976c121656667899899a44ba8373074cdf9a
-  data.tar.gz: 01e93aacc411435f5aa36e6d428508b587bde5ed690e4ba7c35acd3ea08118acbaab376cdc5f8384a73f173227b7b48cb100e76a9c43c3f637a1d55a1bd1aa20
+  metadata.gz: 01c264bdc02ff8b848adc83b7203e1740eed9e99ed310b0162c0d1ef9512e2152641ebbccb435a0b47b98137c859f9e9eaf499f823f377fa178ac61b2581f001
+  data.tar.gz: 76c22db82d0b74b866fe1042d57703e8ea26fed3c8572ea730bfd8f3eb96137b9f10027b31179ad9bf01b5f4e074585e63940d43c64076f2ddcc7e1a1b2ef034

data/CHANGELOG.md

@@ -1,5 +1,24 @@
 # v1.15
 
+## Release v1.15.3 - 2022/11/02
+
+### Bug Fix
+
+* Support glob for `!include` directive in YAML config format
+  https://github.com/fluent/fluentd/pull/3917
+* Remove meaningless oj options
+  https://github.com/fluent/fluentd/pull/3929
+* Fix log initializer to correctly create per-process files on Windows
+  https://github.com/fluent/fluentd/pull/3939
+* out_file: Fix the multi-worker check with `<worker 0-N>` directive
+  https://github.com/fluent/fluentd/pull/3942
+
+### Misc
+
+* Fix broken tests on Ruby 3.2
+  https://github.com/fluent/fluentd/pull/3883
+  https://github.com/fluent/fluentd/pull/3922
+
 ## Release v1.15.2 - 2022/08/22
 
 ### Enhancement

data/lib/fluent/command/fluentd.rb

@@ -345,17 +345,6 @@ end
 exit 0 if early_exit
 
 if opts[:supervise]
-  if Fluent.windows?
-    if opts[:log_path] && opts[:log_path] != "-"
-      if opts[:log_rotate_age] || opts[:log_rotate_size]
-        require 'pathname'
-
-        log_path = Pathname(opts[:log_path]).sub_ext("-supervisor#{Pathname(opts[:log_path]).extname}").to_s
-        opts[:log_path] = log_path
-      end
-    end
-  end
-
   supervisor = Fluent::Supervisor.new(opts)
   supervisor.configure(supervisor: true)
   supervisor.run_supervisor(dry_run: opts[:dry_run])

data/lib/fluent/config/yaml_parser/loader.rb

@@ -44,7 +44,7 @@ module Fluent
           visitor = Visitor.new(scanner, class_loader)
 
           visitor._register_domain(INCLUDE_TAG) do |_, val|
-            load(path.parent.join(val))
+            eval_include(Pathname.new(val), path.parent)
           end
 
           visitor._register_domain(FLUENT_JSON_TAG) do |_, val|
@@ -60,6 +60,23 @@ module Fluent
           end
         end
 
+        def eval_include(path, parent)
+          if path.relative?
+            pattern = parent.join(path)
+          else
+            pattern = path
+          end
+          result = []
+          Dir.glob(pattern).sort.each do |path|
+            result.concat(load(Pathname.new(path)))
+          end
+          result
+        rescue SystemCallError => e
+          parse_error = ConfigParseError.new("include error #{path} - #{e}")
+          parse_error.set_backtrace(e.backtrace)
+          raise parse_error
+        end
+
         class Visitor < Psych::Visitors::ToRuby
           def initialize(scanner, class_loader)
             super(scanner, class_loader)

data/lib/fluent/file_wrapper.rb

@@ -0,0 +1,137 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+unless Fluent.windows?
+  Fluent::FileWrapper = File
+else
+  require 'fluent/win32api'
+
+  module Fluent
+    module FileWrapper
+      def self.open(path, mode='r')
+        io = WindowsFile.new(path, mode).io
+        if block_given?
+          v = yield io
+          io.close
+          v
+        else
+          io
+        end
+      end
+
+      def self.stat(path)
+        f = WindowsFile.new(path)
+        s = f.stat
+        f.close
+        s
+      end
+    end
+
+    class WindowsFile
+      include File::Constants
+
+      attr_reader :io
+
+      INVALID_HANDLE_VALUE = -1
+
+      def initialize(path, mode_enc='r')
+        @path = path
+        mode, enc = mode_enc.split(":", 2)
+        @io = File.open(path, mode2flags(mode))
+        @io.set_encoding(enc) if enc
+        @file_handle = Win32API._get_osfhandle(@io.to_i)
+        @io.instance_variable_set(:@file_index, self.ino)
+        def @io.ino
+          @file_index
+        end
+      end
+
+      def close
+        @io.close
+        @file_handle = INVALID_HANDLE_VALUE
+      end
+
+      # To keep backward compatibility, we continue to use GetFileInformationByHandle()
+      # to get file id.
+      # Note that Ruby's File.stat uses GetFileInformationByHandleEx() with FileIdInfo
+      # and returned value is different with above one, former one is 64 bit while
+      # later one is 128bit.
+      def ino
+        by_handle_file_information = '\0'*(4+8+8+8+4+4+4+4+4+4)   #72bytes
+
+        unless Win32API.GetFileInformationByHandle(@file_handle, by_handle_file_information)
+          return 0
+        end
+
+        by_handle_file_information.unpack("I11Q1")[11] # fileindex
+      end
+
+      def stat
+        raise Errno::ENOENT if delete_pending
+        s = File.stat(@path)
+        s.instance_variable_set :@ino, self.ino
+        def s.ino; @ino; end
+        s
+      end
+
+      private
+
+      def mode2flags(mode)
+        # Always inject File::Constants::SHARE_DELETE
+        # https://github.com/fluent/fluentd/pull/3585#issuecomment-1101502617
+        # To enable SHARE_DELETE, BINARY is also required.
+        # https://bugs.ruby-lang.org/issues/11218
+        # https://github.com/ruby/ruby/blob/d6684f063bc53e3cab025bd39526eca3b480b5e7/win32/win32.c#L6332-L6345
+        flags = BINARY | SHARE_DELETE
+        case mode.delete("b")
+        when "r"
+          flags |= RDONLY
+        when "r+"
+          flags |= RDWR
+        when "w"
+          flags |= WRONLY | CREAT | TRUNC
+        when "w+"
+          flags |= RDWR | CREAT | TRUNC
+        when "a"
+          flags |= WRONLY | CREAT | APPEND
+        when "a+"
+          flags |= RDWR | CREAT | APPEND
+        else
+          raise Errno::EINVAL.new("Unsupported mode by Fluent::FileWrapper: #{mode}")
+        end
+      end
+
+      # DeletePending is a Windows-specific file state that roughly means
+      # "this file is queued for deletion, so close any open handlers"
+      #
+      # This flag can be retrieved via GetFileInformationByHandleEx().
+      #
+      # https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-getfileinformationbyhandleex
+      #
+      def delete_pending
+        file_standard_info = 0x01
+        bufsize = 1024
+        buf = '\0' * bufsize
+
+        unless Win32API.GetFileInformationByHandleEx(@file_handle, file_standard_info, buf, bufsize)
+          return false
+        end
+
+        return buf.unpack("QQICC")[3] != 0
+      end
+    end
+  end
+end

data/lib/fluent/oj_options.rb

@@ -4,14 +4,13 @@ module Fluent
   class OjOptions
     OPTIONS = {
       'bigdecimal_load': :symbol,
-      'max_nesting': :integer,
       'mode': :symbol,
       'use_to_json': :bool
     }
 
     ALLOWED_VALUES = {
       'bigdecimal_load': %i[bigdecimal float auto],
-      'mode': %i[strict null compat json rails object custom]
+      'mode': %i[strict null compat json rails custom]
     }
 
     DEFAULTS = {

data/lib/fluent/plugin/in_tail.rb

@@ -25,12 +25,7 @@ require 'fluent/variable_store'
 require 'fluent/capability'
 require 'fluent/plugin/in_tail/position_file'
 require 'fluent/plugin/in_tail/group_watch'
-
-if Fluent.windows?
-  require_relative 'file_wrapper'
-else
-  Fluent::FileWrapper = File
-end
+require 'fluent/file_wrapper'
 
 module Fluent::Plugin
   class TailInput < Fluent::Plugin::Input

data/lib/fluent/plugin/out_file.rb

@@ -188,10 +188,6 @@ module Fluent::Plugin
         condition = Gem::Dependency.new('', [">= 2.7.0", "< 3.1.0"])
         @need_ruby_on_macos_workaround = true if condition.match?('', RUBY_VERSION)
       end
-
-      if @need_lock && @append && @fluentd_lock_dir.nil?
-        raise Fluent::InvalidLockDirectory, "must set FLUENTD_LOCK_DIR on multi-worker append mode"
-      end
     end
 
     def multi_workers_ready?

data/lib/fluent/supervisor.rb

@@ -525,10 +525,22 @@ module Fluent
         @log_rotate_size = log_rotate_size
       end
 
-      def worker_id_suffixed_path(worker_id, path)
-        require 'pathname'
-
-        Pathname(path).sub_ext("-#{worker_id}#{Pathname(path).extname}").to_s
+      # Create a unique path for each process.
+      #
+      # >>> per_process_path(:worker, 1, "C:/tmp/test.log")
+      # C:/tmp/test-1.log
+      # >>> per_process_path(:supervisor, 0, "C:/tmp/test.log")
+      # C:/tmp/test-supervisor-0.log
+      def self.per_process_path(path, process_type, worker_id)
+        path = Pathname(path)
+        ext = path.extname
+
+        if process_type == :supervisor
+          suffix = "-#{process_type}-0#{ext}"  # "-0" for backword compatibility.
+        else
+          suffix = "-#{worker_id}#{ext}"
+        end
+        return path.sub_ext(suffix).to_s
       end
 
       def init(process_type, worker_id)
@@ -540,13 +552,19 @@ module Fluent
             FileUtils.mkdir_p(File.dirname(@path))
           end
 
-          @logdev = if @log_rotate_age || @log_rotate_size
-                     Fluent::LogDeviceIO.new(Fluent.windows? ?
-                                               worker_id_suffixed_path(worker_id, @path) : @path,
-                                             shift_age: @log_rotate_age, shift_size: @log_rotate_size)
-                   else
-                     File.open(@path, "a")
-                   end
+          if @log_rotate_age || @log_rotate_size
+            # We need to prepare a unique path for each worker since
+            # Windows locks files.
+            if Fluent.windows?
+                path = LoggerInitializer.per_process_path(@path, process_type, worker_id)
+            else
+                path = @path
+            end
+            @logdev = Fluent::LogDeviceIO.new(path, shift_age: @log_rotate_age, shift_size: @log_rotate_size)
+          else
+            @logdev = File.open(@path, "a")
+          end
+
           if @chuser || @chgroup
             chuid = @chuser ? ServerEngine::Privilege.get_etc_passwd(@chuser).uid : nil
             chgid = @chgroup ? ServerEngine::Privilege.get_etc_group(@chgroup).gid : nil
@@ -565,6 +583,7 @@ module Fluent
         $log = Fluent::Log.new(logger, @opts)
         $log.enable_color(false) if @path
         $log.enable_debug if @level <= Fluent::Log::LEVEL_DEBUG
+        $log.info "init #{process_type} logger", path: path, rotate_age: @log_rotate_age, rotate_size: @log_rotate_size
       end
 
       def stdout?

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.15.2'
+  VERSION = '1.15.3'
 
 end

data/test/command/test_fluentd.rb

@@ -5,20 +5,35 @@ require_relative '../helper'
 
 require 'fileutils'
 require 'timeout'
+require 'securerandom'
+require 'fluent/file_wrapper'
 
 class TestFluentdCommand < ::Test::Unit::TestCase
-  TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/command/fluentd#{ENV['TEST_ENV_NUMBER']}")
   SUPERVISOR_PID_PATTERN = /starting fluentd-[.0-9]+ pid=(\d+)/
   WORKER_PID_PATTERN = /starting fluentd worker pid=(\d+) /
 
+  def tmp_dir
+    File.join(File.dirname(__FILE__), "..", "tmp", "command" "fluentd#{ENV['TEST_ENV_NUMBER']}", SecureRandom.hex(10))
+  end
+
   setup do
-    FileUtils.rm_rf(TMP_DIR)
-    FileUtils.mkdir_p(TMP_DIR)
+    @tmp_dir = tmp_dir
+    FileUtils.mkdir_p(@tmp_dir)
     @supervisor_pid = nil
     @worker_pids = []
     ENV["TEST_RUBY_PATH"] = nil
   end
 
+  teardown do
+    begin
+      FileUtils.rm_rf(@tmp_dir)
+    rescue Errno::EACCES
+      # It may occur on Windows because of delete pending state due to delayed GC.
+      # Ruby 3.2 or later doesn't ignore Errno::EACCES:
+      # https://github.com/ruby/ruby/commit/983115cf3c8f75b1afbe3274f02c1529e1ce3a81
+    end
+  end
+
   def process_exist?(pid)
     begin
       r = Process.waitpid(pid, Process::WNOHANG)
@@ -30,17 +45,17 @@ class TestFluentdCommand < ::Test::Unit::TestCase
   end
 
   def create_conf_file(name, content, ext_enc = 'utf-8')
-    conf_path = File.join(TMP_DIR, name)
-    File.open(conf_path, "w:#{ext_enc}:utf-8") do |file|
+    conf_path = File.join(@tmp_dir, name)
+    Fluent::FileWrapper.open(conf_path, "w:#{ext_enc}:utf-8") do |file|
       file.write content
     end
     conf_path
   end
 
   def create_plugin_file(name, content)
-    file_path = File.join(TMP_DIR, 'plugin', name)
+    file_path = File.join(@tmp_dir, 'plugin', name)
     FileUtils.mkdir_p(File.dirname(file_path))
-    File.open(file_path, 'w') do |file|
+    Fluent::FileWrapper.open(file_path, 'w') do |file|
       file.write content
     end
     file_path
@@ -56,8 +71,8 @@ class TestFluentdCommand < ::Test::Unit::TestCase
     end
   end
 
-  def execute_command(cmdline, chdir=TMP_DIR, env = {})
-    null_stream = File.open(File::NULL, 'w')
+  def execute_command(cmdline, chdir=@tmp_dir, env = {})
+    null_stream = Fluent::FileWrapper.open(File::NULL, 'w')
     gemfile_path = File.expand_path(File.dirname(__FILE__) + "../../../Gemfile")
 
     env = { "BUNDLE_GEMFILE" => gemfile_path }.merge(env)
@@ -103,7 +118,7 @@ class TestFluentdCommand < ::Test::Unit::TestCase
     assert_error_msg = ""
     stdio_buf = ""
     begin
-      execute_command(cmdline, TMP_DIR, env) do |pid, stdout|
+      execute_command(cmdline, @tmp_dir, env) do |pid, stdout|
         begin
           waiting(timeout) do
             while process_exist?(pid) && !matched
@@ -269,7 +284,7 @@ CONF
 
   sub_test_case 'with system configuration about root directory' do
     setup do
-      @root_path = File.join(TMP_DIR, "rootpath")
+      @root_path = File.join(@tmp_dir, "rootpath")
       FileUtils.rm_rf(@root_path)
       @conf = <<CONF
 <system>
@@ -308,7 +323,7 @@ CONF
     end
 
     test 'fails to launch fluentd if specified root path is invalid path for directory' do
-      File.open(@root_path, 'w') do |_|
+      Fluent::FileWrapper.open(@root_path, 'w') do |_|
         # create file and close it
       end
       conf_path = create_conf_file('existing_root_dir.conf', @conf)
@@ -508,7 +523,7 @@ CONF
 
       assert_fluentd_fails_to_start(
         create_cmdline(conf_path, "-p", File.dirname(plugin_path)),
-        "in_buggy.rb:5: syntax error, unexpected end-of-input, expecting"
+        "in_buggy.rb:5: syntax error, unexpected end-of-input"
       )
     end
   end
@@ -554,7 +569,7 @@ CONF
 
   sub_test_case 'configured to run 2 workers' do
     setup do
-      @root_path = File.join(TMP_DIR, "rootpath")
+      @root_path = File.join(@tmp_dir, "rootpath")
       FileUtils.rm_rf(@root_path)
       FileUtils.mkdir_p(@root_path)
     end
@@ -961,10 +976,10 @@ CONF
 </match>
 CONF
       ruby_path = ServerEngine.ruby_bin_path
-      tmp_ruby_path = File.join(TMP_DIR, "ruby with spaces")
+      tmp_ruby_path = File.join(@tmp_dir, "ruby with spaces")
       if Fluent.windows?
         tmp_ruby_path << ".bat"
-        File.open(tmp_ruby_path, "w") do |file|
+        Fluent::FileWrapper.open(tmp_ruby_path, "w") do |file|
           file.write "#{ruby_path} %*"
         end
       else