fluentd 1.15.1-x64-mingw32 → 1.15.3-x64-mingw32

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eeb9d264af514e920d3edf0d5520b3c26588fb4bdcf7166f6a5bbf8b9bd9a4c5
-  data.tar.gz: 4273c9f6178dc0bb04ac79433c12fc60a0b2ebe5a5876f7fe39b1c4278cae2ad
+  metadata.gz: e61469818899ba24ff0ab86d7ea73f28693254c48b55285bf91154472f18f2ed
+  data.tar.gz: 9a90482923948c6b29f6628a1854db956e6838f54a281e4627497f8e9acd67c6
 SHA512:
-  metadata.gz: 0cab9a5875bfb3cd2813727201c4fe82e4318f3fac74d84c526c1f38925a46a7c8d5ccc6b18d272280f471c39bbb235885b0e7931c92c19f06e99e36d3929db3
-  data.tar.gz: bd4ef3aacb731e0f8eeb013b52160e40225f57d15f698fbb8b034e79fbfff2ae3567c76bb5f7d9302b01fff1dca28537886d254a94e5a3af4d913c580636a35d
+  metadata.gz: ecc1b848b5189e77eca3b7badad98da789c08eaafc8a029355d1757ba9164d66765571fae98cf1765b9d60f6168db39b798dbd0b7a0c564eee45a4a191e261c1
+  data.tar.gz: 76c22db82d0b74b866fe1042d57703e8ea26fed3c8572ea730bfd8f3eb96137b9f10027b31179ad9bf01b5f4e074585e63940d43c64076f2ddcc7e1a1b2ef034

data/CHANGELOG.md

@@ -1,5 +1,44 @@
 # v1.15
 
+## Release v1.15.3 - 2022/11/02
+
+### Bug Fix
+
+* Support glob for `!include` directive in YAML config format
+  https://github.com/fluent/fluentd/pull/3917
+* Remove meaningless oj options
+  https://github.com/fluent/fluentd/pull/3929
+* Fix log initializer to correctly create per-process files on Windows
+  https://github.com/fluent/fluentd/pull/3939
+* out_file: Fix the multi-worker check with `<worker 0-N>` directive
+  https://github.com/fluent/fluentd/pull/3942
+
+### Misc
+
+* Fix broken tests on Ruby 3.2
+  https://github.com/fluent/fluentd/pull/3883
+  https://github.com/fluent/fluentd/pull/3922
+
+## Release v1.15.2 - 2022/08/22
+
+### Enhancement
+
+* Add a new system configuration `enable_jit`
+  https://github.com/fluent/fluentd/pull/3857
+
+### Bug Fix
+
+* out_file: Fix append mode with `--daemon` flag
+  https://github.com/fluent/fluentd/pull/3864
+* child_process: Plug file descriptor leak
+  https://github.com/fluent/fluentd/pull/3844
+
+### Misc
+
+* Drop win32-api gem to support Ruby 3.2
+  https://github.com/fluent/fluentd/pull/3849
+  https://github.com/fluent/fluentd/pull/3866
+
 ## Release v1.15.1 - 2022/07/27
 
 ### Bug Fix

data/fluentd.gemspec

@@ -34,12 +34,9 @@ Gem::Specification.new do |gem|
   fake_platform = ENV['GEM_BUILD_FAKE_PLATFORM'].to_s
   gem.platform = fake_platform unless fake_platform.empty?
   if /mswin|mingw/ =~ fake_platform || (/mswin|mingw/ =~ RUBY_PLATFORM && fake_platform.empty?)
-    gem.add_runtime_dependency("win32-api", [">= 1.10", "< 2.0.0"])
     gem.add_runtime_dependency("win32-service", ["~> 2.3.0"])
     gem.add_runtime_dependency("win32-ipc", ["~> 0.7.0"])
     gem.add_runtime_dependency("win32-event", ["~> 0.6.3"])
-    gem.add_runtime_dependency("windows-api", ["~> 0.4.5"])
-    gem.add_runtime_dependency("windows-pr", ["~> 1.2.6"])
     gem.add_runtime_dependency("certstore_c", ["~> 0.1.7"])
   end
 

data/lib/fluent/command/fluentd.rb

@@ -189,9 +189,6 @@ op.on('--disable-shared-socket', "Don't open shared socket for multiple workers"
 }
 
 if Fluent.windows?
-  require 'windows/library'
-  include Windows::Library
-
   opts.merge!(
     :winsvc_name => 'fluentdwinsvc',
     :winsvc_display_name => 'Fluentd Windows Service',
@@ -292,9 +289,7 @@ if winsvcinstmode = opts[:regwinsvc]
   case winsvcinstmode
   when 'i'
     binary_path = File.join(File.dirname(__FILE__), "..")
-    ruby_path = "\0" * 256
-    GetModuleFileName.call(0,ruby_path,256)
-    ruby_path = ruby_path.rstrip.gsub(/\\/, '/')
+    ruby_path = ServerEngine.ruby_bin_path
     start_type = Service::DEMAND_START
     if opts[:regwinsvcautostart]
       start_type = Service::AUTO_START
@@ -350,17 +345,6 @@ end
 exit 0 if early_exit
 
 if opts[:supervise]
-  if Fluent.windows?
-    if opts[:log_path] && opts[:log_path] != "-"
-      if opts[:log_rotate_age] || opts[:log_rotate_size]
-        require 'pathname'
-
-        log_path = Pathname(opts[:log_path]).sub_ext("-supervisor#{Pathname(opts[:log_path]).extname}").to_s
-        opts[:log_path] = log_path
-      end
-    end
-  end
-
   supervisor = Fluent::Supervisor.new(opts)
   supervisor.configure(supervisor: true)
   supervisor.run_supervisor(dry_run: opts[:dry_run])

data/lib/fluent/config/yaml_parser/loader.rb

@@ -44,7 +44,7 @@ module Fluent
           visitor = Visitor.new(scanner, class_loader)
 
           visitor._register_domain(INCLUDE_TAG) do |_, val|
-            load(path.parent.join(val))
+            eval_include(Pathname.new(val), path.parent)
           end
 
           visitor._register_domain(FLUENT_JSON_TAG) do |_, val|
@@ -60,6 +60,23 @@ module Fluent
           end
         end
 
+        def eval_include(path, parent)
+          if path.relative?
+            pattern = parent.join(path)
+          else
+            pattern = path
+          end
+          result = []
+          Dir.glob(pattern).sort.each do |path|
+            result.concat(load(Pathname.new(path)))
+          end
+          result
+        rescue SystemCallError => e
+          parse_error = ConfigParseError.new("include error #{path} - #{e}")
+          parse_error.set_backtrace(e.backtrace)
+          raise parse_error
+        end
+
         class Visitor < Psych::Visitors::ToRuby
           def initialize(scanner, class_loader)
             super(scanner, class_loader)

data/lib/fluent/file_wrapper.rb

@@ -0,0 +1,137 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+unless Fluent.windows?
+  Fluent::FileWrapper = File
+else
+  require 'fluent/win32api'
+
+  module Fluent
+    module FileWrapper
+      def self.open(path, mode='r')
+        io = WindowsFile.new(path, mode).io
+        if block_given?
+          v = yield io
+          io.close
+          v
+        else
+          io
+        end
+      end
+
+      def self.stat(path)
+        f = WindowsFile.new(path)
+        s = f.stat
+        f.close
+        s
+      end
+    end
+
+    class WindowsFile
+      include File::Constants
+
+      attr_reader :io
+
+      INVALID_HANDLE_VALUE = -1
+
+      def initialize(path, mode_enc='r')
+        @path = path
+        mode, enc = mode_enc.split(":", 2)
+        @io = File.open(path, mode2flags(mode))
+        @io.set_encoding(enc) if enc
+        @file_handle = Win32API._get_osfhandle(@io.to_i)
+        @io.instance_variable_set(:@file_index, self.ino)
+        def @io.ino
+          @file_index
+        end
+      end
+
+      def close
+        @io.close
+        @file_handle = INVALID_HANDLE_VALUE
+      end
+
+      # To keep backward compatibility, we continue to use GetFileInformationByHandle()
+      # to get file id.
+      # Note that Ruby's File.stat uses GetFileInformationByHandleEx() with FileIdInfo
+      # and returned value is different with above one, former one is 64 bit while
+      # later one is 128bit.
+      def ino
+        by_handle_file_information = '\0'*(4+8+8+8+4+4+4+4+4+4)   #72bytes
+
+        unless Win32API.GetFileInformationByHandle(@file_handle, by_handle_file_information)
+          return 0
+        end
+
+        by_handle_file_information.unpack("I11Q1")[11] # fileindex
+      end
+
+      def stat
+        raise Errno::ENOENT if delete_pending
+        s = File.stat(@path)
+        s.instance_variable_set :@ino, self.ino
+        def s.ino; @ino; end
+        s
+      end
+
+      private
+
+      def mode2flags(mode)
+        # Always inject File::Constants::SHARE_DELETE
+        # https://github.com/fluent/fluentd/pull/3585#issuecomment-1101502617
+        # To enable SHARE_DELETE, BINARY is also required.
+        # https://bugs.ruby-lang.org/issues/11218
+        # https://github.com/ruby/ruby/blob/d6684f063bc53e3cab025bd39526eca3b480b5e7/win32/win32.c#L6332-L6345
+        flags = BINARY | SHARE_DELETE
+        case mode.delete("b")
+        when "r"
+          flags |= RDONLY
+        when "r+"
+          flags |= RDWR
+        when "w"
+          flags |= WRONLY | CREAT | TRUNC
+        when "w+"
+          flags |= RDWR | CREAT | TRUNC
+        when "a"
+          flags |= WRONLY | CREAT | APPEND
+        when "a+"
+          flags |= RDWR | CREAT | APPEND
+        else
+          raise Errno::EINVAL.new("Unsupported mode by Fluent::FileWrapper: #{mode}")
+        end
+      end
+
+      # DeletePending is a Windows-specific file state that roughly means
+      # "this file is queued for deletion, so close any open handlers"
+      #
+      # This flag can be retrieved via GetFileInformationByHandleEx().
+      #
+      # https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-getfileinformationbyhandleex
+      #
+      def delete_pending
+        file_standard_info = 0x01
+        bufsize = 1024
+        buf = '\0' * bufsize
+
+        unless Win32API.GetFileInformationByHandleEx(@file_handle, file_standard_info, buf, bufsize)
+          return false
+        end
+
+        return buf.unpack("QQICC")[3] != 0
+      end
+    end
+  end
+end

data/lib/fluent/oj_options.rb

@@ -4,14 +4,13 @@ module Fluent
   class OjOptions
     OPTIONS = {
       'bigdecimal_load': :symbol,
-      'max_nesting': :integer,
       'mode': :symbol,
       'use_to_json': :bool
     }
 
     ALLOWED_VALUES = {
       'bigdecimal_load': %i[bigdecimal float auto],
-      'mode': %i[strict null compat json rails object custom]
+      'mode': %i[strict null compat json rails custom]
     }
 
     DEFAULTS = {

data/lib/fluent/plugin/in_tail.rb

@@ -25,12 +25,7 @@ require 'fluent/variable_store'
 require 'fluent/capability'
 require 'fluent/plugin/in_tail/position_file'
 require 'fluent/plugin/in_tail/group_watch'
-
-if Fluent.windows?
-  require_relative 'file_wrapper'
-else
-  Fluent::FileWrapper = File
-end
+require 'fluent/file_wrapper'
 
 module Fluent::Plugin
   class TailInput < Fluent::Plugin::Input

data/lib/fluent/plugin/out_file.rb

@@ -188,10 +188,6 @@ module Fluent::Plugin
         condition = Gem::Dependency.new('', [">= 2.7.0", "< 3.1.0"])
         @need_ruby_on_macos_workaround = true if condition.match?('', RUBY_VERSION)
       end
-
-      if @need_lock && @append && @fluentd_lock_dir.nil?
-        raise Fluent::InvalidLockDirectory, "must set FLUENTD_LOCK_DIR on multi-worker append mode"
-      end
     end
 
     def multi_workers_ready?

data/lib/fluent/plugin_helper/child_process.rb

@@ -346,6 +346,9 @@ module Fluent
           if cb
             cb.call(process_info.exit_status) rescue nil
           end
+          process_info.readio&.close rescue nil
+          process_info.writeio&.close rescue nil
+          process_info.stderrio&.close rescue nil
         end
         thread[:_fluentd_plugin_helper_child_process_running] = true
         thread[:_fluentd_plugin_helper_child_process_pid] = pid

data/lib/fluent/supervisor.rb

@@ -32,12 +32,6 @@ require 'fluent/variable_store'
 require 'serverengine'
 
 if Fluent.windows?
-  require 'windows/library'
-  require 'windows/synchronize'
-  require 'windows/system_info'
-  include Windows::Library
-  include Windows::Synchronize
-  include Windows::SystemInfo
   require 'win32/ipc'
   require 'win32/event'
 end
@@ -50,6 +44,9 @@ module Fluent
       @rpc_server = nil
       @counter = nil
 
+      @fluentd_lock_dir = Dir.mktmpdir("fluentd-lock-")
+      ENV['FLUENTD_LOCK_DIR'] = @fluentd_lock_dir
+
       if config[:rpc_endpoint]
         @rpc_endpoint = config[:rpc_endpoint]
         @enable_get_dump = config[:enable_get_dump]
@@ -79,9 +76,15 @@ module Fluent
       stop_windows_event_thread if Fluent.windows?
       stop_rpc_server if @rpc_endpoint
       stop_counter_server if @counter
+      cleanup_lock_dir
       Fluent::Supervisor.cleanup_resources
     end
 
+    def cleanup_lock_dir
+      FileUtils.rm(Dir.glob(File.join(@fluentd_lock_dir, "fluentd-*.lock")))
+      FileUtils.rmdir(@fluentd_lock_dir)
+    end
+
     def run_rpc_server
       @rpc_server = RPC::Server.new(@rpc_endpoint, $log)
 
@@ -235,7 +238,8 @@ module Fluent
         end
         begin
           loop do
-            ipc_idx = ipc.wait_any(events.map {|e| e[:win32_event]}, Windows::Synchronize::INFINITE)
+            infinite = 0xFFFFFFFF
+            ipc_idx = ipc.wait_any(events.map {|e| e[:win32_event]}, infinite)
             event_idx = ipc_idx - 1
 
             if event_idx >= 0 && event_idx < events.length
@@ -521,10 +525,22 @@ module Fluent
         @log_rotate_size = log_rotate_size
       end
 
-      def worker_id_suffixed_path(worker_id, path)
-        require 'pathname'
-
-        Pathname(path).sub_ext("-#{worker_id}#{Pathname(path).extname}").to_s
+      # Create a unique path for each process.
+      #
+      # >>> per_process_path(:worker, 1, "C:/tmp/test.log")
+      # C:/tmp/test-1.log
+      # >>> per_process_path(:supervisor, 0, "C:/tmp/test.log")
+      # C:/tmp/test-supervisor-0.log
+      def self.per_process_path(path, process_type, worker_id)
+        path = Pathname(path)
+        ext = path.extname
+
+        if process_type == :supervisor
+          suffix = "-#{process_type}-0#{ext}"  # "-0" for backword compatibility.
+        else
+          suffix = "-#{worker_id}#{ext}"
+        end
+        return path.sub_ext(suffix).to_s
       end
 
       def init(process_type, worker_id)
@@ -536,13 +552,19 @@ module Fluent
             FileUtils.mkdir_p(File.dirname(@path))
           end
 
-          @logdev = if @log_rotate_age || @log_rotate_size
-                     Fluent::LogDeviceIO.new(Fluent.windows? ?
-                                               worker_id_suffixed_path(worker_id, @path) : @path,
-                                             shift_age: @log_rotate_age, shift_size: @log_rotate_size)
-                   else
-                     File.open(@path, "a")
-                   end
+          if @log_rotate_age || @log_rotate_size
+            # We need to prepare a unique path for each worker since
+            # Windows locks files.
+            if Fluent.windows?
+                path = LoggerInitializer.per_process_path(@path, process_type, worker_id)
+            else
+                path = @path
+            end
+            @logdev = Fluent::LogDeviceIO.new(path, shift_age: @log_rotate_age, shift_size: @log_rotate_size)
+          else
+            @logdev = File.open(@path, "a")
+          end
+
           if @chuser || @chgroup
             chuid = @chuser ? ServerEngine::Privilege.get_etc_passwd(@chuser).uid : nil
             chgid = @chgroup ? ServerEngine::Privilege.get_etc_group(@chgroup).gid : nil
@@ -561,6 +583,7 @@ module Fluent
         $log = Fluent::Log.new(logger, @opts)
         $log.enable_color(false) if @path
         $log.enable_debug if @level <= Fluent::Log::LEVEL_DEBUG
+        $log.info "init #{process_type} logger", path: path, rotate_age: @log_rotate_age, rotate_size: @log_rotate_size
       end
 
       def stdout?
@@ -876,10 +899,7 @@ module Fluent
         Fluent::Supervisor.load_config(@config_path, params)
       }
 
-      Dir.mktmpdir("fluentd-lock-") do |lock_dir|
-        ENV['FLUENTD_LOCK_DIR'] = lock_dir
-        se.run
-      end
+      se.run
     end
 
     def install_main_process_signal_handlers
@@ -1104,6 +1124,11 @@ module Fluent
         fluentd_spawn_cmd << '-Eascii-8bit:ascii-8bit'
       end
 
+      if @system_config.enable_jit
+        $log.info "enable Ruby JIT for workers (--jit)"
+        fluentd_spawn_cmd << '--jit'
+      end
+
       # Adding `-h` so that it can avoid ruby's command blocking
       # e.g. `ruby -Eascii-8bit:ascii-8bit` will block. but `ruby -Eascii-8bit:ascii-8bit -h` won't.
       _, e, s = Open3.capture3(*fluentd_spawn_cmd, "-h")

data/lib/fluent/system_config.rb

@@ -28,7 +28,7 @@ module Fluent
       :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
       :file_permission, :dir_permission, :counter_server, :counter_client,
       :strict_config_value, :enable_msgpack_time_support, :disable_shared_socket,
-      :metrics, :enable_input_metrics, :enable_size_metrics
+      :metrics, :enable_input_metrics, :enable_size_metrics, :enable_jit
     ]
 
     config_param :workers,   :integer, default: 1
@@ -50,6 +50,7 @@ module Fluent
     config_param :disable_shared_socket, :bool, default: nil
     config_param :enable_input_metrics, :bool, default: nil
     config_param :enable_size_metrics, :bool, default: nil
+    config_param :enable_jit, :bool, default: false
     config_param :file_permission, default: nil do |v|
       v.to_i(8)
     end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.15.1'
+  VERSION = '1.15.3'
 
 end

data/lib/fluent/win32api.rb

@@ -0,0 +1,38 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/env'
+
+module Fluent
+  module Win32API
+    require 'fiddle/import'
+    require 'fiddle/types'
+    extend Fiddle::Importer
+
+    if RUBY_PLATFORM.split('-')[-1] == "ucrt"
+      MSVCRT_DLL = 'ucrtbase.dll'
+    else
+      MSVCRT_DLL = 'msvcrt.dll'
+    end
+
+    dlload MSVCRT_DLL, "kernel32.dll"
+    include Fiddle::Win32Types
+
+    extern "intptr_t _get_osfhandle(int)"
+    extern "BOOL GetFileInformationByHandle(HANDLE, void *)"
+    extern "BOOL GetFileInformationByHandleEx(HANDLE, int, void *, DWORD)"
+  end if Fluent.windows?
+end

data/lib/fluent/winsvc.rb

@@ -17,14 +17,12 @@
 begin
 
   require 'optparse'
-  require 'windows/debug'
-  require 'Windows/Library'
   require 'win32/daemon'
   require 'win32/event'
+  require 'win32/registry'
+  require 'serverengine'
 
   include Win32
-  include Windows::Library
-  include Windows::Debug
 
   op = OptionParser.new
   opts = {service_name: nil}
@@ -37,16 +35,13 @@ begin
   end 
 
   def read_fluentdopt(service_name)
-    require 'win32/Registry'
     Win32::Registry::HKEY_LOCAL_MACHINE.open("SYSTEM\\CurrentControlSet\\Services\\#{service_name}") do |reg|
       reg.read("fluentdopt")[1] rescue ""
     end
   end
 
   def service_main_start(service_name)
-    ruby_path = 0.chr * 260
-    GetModuleFileName.call(0, ruby_path,260)
-    ruby_path = ruby_path.rstrip.gsub(/\\/, '/')
+    ruby_path = ServerEngine.ruby_bin_path
     rubybin_dir = ruby_path[0, ruby_path.rindex("/")]
     opt = read_fluentdopt(service_name)
     Process.spawn("\"#{rubybin_dir}/ruby.exe\" \"#{rubybin_dir}/fluentd\" #{opt} -x #{service_name}")