fluentd 1.15.2 → 1.15.3

data/test/plugin/test_out_file.rb

@@ -5,6 +5,7 @@ require 'fileutils'
 require 'time'
 require 'timecop'
 require 'zlib'
+require 'fluent/file_wrapper'
 
 class FileOutputTest < Test::Unit::TestCase
   def setup
@@ -1016,7 +1017,7 @@ class FileOutputTest < Test::Unit::TestCase
 
     test 'returns filepath with index which does not exist yet' do
       5.times do |i|
-        File.open(File.join(@tmp, "exist_#{i}.log"), 'a'){|f| } # open(create) and close
+        Fluent::FileWrapper.open(File.join(@tmp, "exist_#{i}.log"), 'a'){|f| } # open(create) and close
       end
       @i.find_filepath_available(File.join(@tmp, "exist_**.log")) do |path|
         assert_equal File.join(@tmp, "exist_5.log"), path
@@ -1025,7 +1026,7 @@ class FileOutputTest < Test::Unit::TestCase
 
     test 'creates lock directory when with_lock is true to exclude operations of other worker process' do
       5.times do |i|
-        File.open(File.join(@tmp, "exist_#{i}.log"), 'a')
+        Fluent::FileWrapper.open(File.join(@tmp, "exist_#{i}.log"), 'a')
       end
       Dir.mkdir(File.join(@tmp, "exist_5.log.lock"))
       @i.find_filepath_available(File.join(@tmp, "exist_**.log"), with_lock: true) do |path|

data/test/test_config.rb

@@ -237,6 +237,63 @@ class ConfigTest < Test::Unit::TestCase
         ])
     end
 
+    def test_included_glob
+      write_config "#{TMP_DIR}/config.yaml", <<-EOS
+      config:
+        - !include "include/*.yaml"
+      EOS
+      write_config "#{TMP_DIR}/include/02_source2.yaml", <<-EOS
+      - source:
+          $type: dummy
+          tag: tag.dummy
+      EOS
+      write_config "#{TMP_DIR}/include/01_source1.yaml", <<-EOS
+      - source:
+          $type: tcp
+          tag: tag.tcp
+          parse:
+            $arg:
+              - why.parse.section.doesnot.have.arg
+              - huh
+            $type: none
+      EOS
+      write_config "#{TMP_DIR}/include/03_match1.yaml", <<-EOS
+      - match:
+          $tag: tag.*
+          $type: stdout
+          buffer:
+            $type: memory
+            flush_interval: 1s
+      EOS
+      root_conf = read_config("#{TMP_DIR}/config.yaml", use_yaml: true)
+      tcp_source_conf = root_conf.elements.first
+      dummy_source_conf = root_conf.elements[1]
+      parse_tcp_conf = tcp_source_conf.elements.first
+      match_conf = root_conf.elements[2]
+
+      assert_equal(
+        [
+          'tcp',
+          'tag.tcp',
+          'none',
+          'why.parse.section.doesnot.have.arg,huh',
+          'dummy',
+          'tag.dummy',
+          'stdout',
+          'tag.*',
+        ],
+        [
+          tcp_source_conf['@type'],
+          tcp_source_conf['tag'],
+          parse_tcp_conf['@type'],
+          parse_tcp_conf.arg,
+          dummy_source_conf['@type'],
+          dummy_source_conf['tag'],
+          match_conf['@type'],
+          match_conf.arg,
+        ])
+    end
+
     def test_check_not_fetchd
       write_config "#{TMP_DIR}/config_test_not_fetched.yaml", <<-EOS
       config:

data/test/{plugin/test_file_wrapper.rb → test_file_wrapper.rb}

@@ -1,5 +1,5 @@
-require_relative '../helper'
-require 'fluent/plugin/file_wrapper'
+require_relative 'helper'
+require 'fluent/file_wrapper'
 
 class FileWrapperTest < Test::Unit::TestCase
   TMP_DIR = File.dirname(__FILE__) + "/../tmp/file_wrapper#{ENV['TEST_ENV_NUMBER']}"

data/test/test_log.rb

@@ -4,13 +4,16 @@ require 'fluent/engine'
 require 'fluent/log'
 require 'timecop'
 require 'logger'
+require 'securerandom'
 
 class LogTest < Test::Unit::TestCase
-  TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/tmp/log/#{ENV['TEST_ENV_NUMBER']}")
+  def tmp_dir
+    File.join(File.dirname(__FILE__), "tmp", "log", "#{ENV['TEST_ENV_NUMBER']}", SecureRandom.hex(10))
+  end
 
   def setup
-    FileUtils.rm_rf(TMP_DIR)
-    FileUtils.mkdir_p(TMP_DIR)
+    @tmp_dir = tmp_dir
+    FileUtils.mkdir_p(@tmp_dir)
     @log_device = Fluent::Test::DummyLogDevice.new
     @timestamp = Time.parse("2016-04-21 02:58:41 +0000")
     @timestamp_str = @timestamp.strftime("%Y-%m-%d %H:%M:%S %z")
@@ -21,6 +24,13 @@ class LogTest < Test::Unit::TestCase
     @log_device.reset
     Timecop.return
     Thread.current[:last_repeated_stacktrace] = nil
+    begin
+      FileUtils.rm_rf(@tmp_dir)
+    rescue Errno::EACCES
+      # It may occur on Windows because of delete pending state due to delayed GC.
+      # Ruby 3.2 or later doesn't ignore Errno::EACCES:
+      # https://github.com/ruby/ruby/commit/983115cf3c8f75b1afbe3274f02c1529e1ce3a81
+    end
   end
 
   sub_test_case "log level" do
@@ -560,7 +570,7 @@ class LogTest < Test::Unit::TestCase
       Timecop.freeze(@timestamp)
 
       rotate_age, rotate_size, travel_term = expected
-      path = "#{TMP_DIR}/log-dev-io-#{rotate_size}-#{rotate_age}"
+      path = "#{@tmp_dir}/log-dev-io-#{rotate_size}-#{rotate_age}"
 
       logdev = Fluent::LogDeviceIO.new(path, shift_age: rotate_age, shift_size: rotate_size)
       logger = ServerEngine::DaemonLogger.new(logdev)
@@ -585,7 +595,7 @@ class LogTest < Test::Unit::TestCase
     with_timezone('utc') do
       rotate_age = 2
       rotate_size = 100
-      path = "#{TMP_DIR}/log-dev-io-#{rotate_size}-#{rotate_age}"
+      path = "#{@tmp_dir}/log-dev-io-#{rotate_size}-#{rotate_age}"
       path0 = path + '.0'
       path1 = path + '.1'
 

data/test/test_supervisor.rb

@@ -2,12 +2,14 @@ require_relative 'helper'
 require 'fluent/event_router'
 require 'fluent/system_config'
 require 'fluent/supervisor'
+require 'fluent/file_wrapper'
 require_relative 'test_plugin_classes'
 
 require 'net/http'
 require 'uri'
 require 'fileutils'
 require 'tempfile'
+require 'securerandom'
 
 if Fluent.windows?
   require 'win32/event'
@@ -22,17 +24,29 @@ class SupervisorTest < ::Test::Unit::TestCase
     end
   end
 
-  TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/tmp/supervisor#{ENV['TEST_ENV_NUMBER']}")
-  TMP_ROOT_DIR = File.join(TMP_DIR, 'root')
+  def tmp_dir
+    File.join(File.dirname(__FILE__), "tmp", "supervisor#{ENV['TEST_ENV_NUMBER']}", SecureRandom.hex(10))
+  end
 
   def setup
-    FileUtils.rm_rf(TMP_DIR)
-    FileUtils.mkdir_p(TMP_DIR)
+    @tmp_dir = tmp_dir
+    @tmp_root_dir = File.join(@tmp_dir, 'root')
+    FileUtils.mkdir_p(@tmp_dir)
+  end
+
+  def teardown
+    begin
+      FileUtils.rm_rf(@tmp_dir)
+    rescue Errno::EACCES
+      # It may occur on Windows because of delete pending state due to delayed GC.
+      # Ruby 3.2 or later doesn't ignore Errno::EACCES:
+      # https://github.com/ruby/ruby/commit/983115cf3c8f75b1afbe3274f02c1529e1ce3a81
+    end
   end
 
   def write_config(path, data)
     FileUtils.mkdir_p(File.dirname(path))
-    File.open(path, "w") {|f| f.write data }
+    Fluent::FileWrapper.open(path, "w") {|f| f.write data }
   end
 
 
@@ -48,7 +62,7 @@ class SupervisorTest < ::Test::Unit::TestCase
   enable_get_dump true
   process_name "process_name"
   log_level info
-  root_dir #{TMP_ROOT_DIR}
+  root_dir #{@tmp_root_dir}
   <log>
     format json
     time_format %Y
@@ -76,7 +90,7 @@ class SupervisorTest < ::Test::Unit::TestCase
     assert_equal true, sys_conf.enable_get_dump
     assert_equal "process_name", sys_conf.process_name
     assert_equal 2, sys_conf.log_level
-    assert_equal TMP_ROOT_DIR, sys_conf.root_dir
+    assert_equal @tmp_root_dir, sys_conf.root_dir
     assert_equal :json, sys_conf.log.format
     assert_equal '%Y', sys_conf.log.time_format
     counter_server = sys_conf.counter_server
@@ -116,7 +130,7 @@ class SupervisorTest < ::Test::Unit::TestCase
         enable_get_dump: true
         process_name: "process_name"
         log_level: info
-        root_dir: !fluent/s "#{TMP_ROOT_DIR}"
+        root_dir: !fluent/s "#{@tmp_root_dir}"
         log:
           format: json
           time_format: "%Y"
@@ -144,7 +158,7 @@ class SupervisorTest < ::Test::Unit::TestCase
         true,
         "process_name",
         2,
-        TMP_ROOT_DIR,
+        @tmp_root_dir,
         :json,
         '%Y',
         '127.0.0.1',
@@ -449,7 +463,7 @@ class SupervisorTest < ::Test::Unit::TestCase
   end
 
   def test_load_config
-    tmp_dir = "#{TMP_DIR}/dir/test_load_config.conf"
+    tmp_dir = "#{@tmp_dir}/dir/test_load_config.conf"
     conf_info_str = %[
 <system>
   log_level info
@@ -520,7 +534,7 @@ class SupervisorTest < ::Test::Unit::TestCase
   end
 
   def test_load_config_for_logger
-    tmp_dir = "#{TMP_DIR}/dir/test_load_config_log.conf"
+    tmp_dir = "#{@tmp_dir}/dir/test_load_config_log.conf"
     conf_info_str = %[
 <system>
   <log>
@@ -547,7 +561,7 @@ class SupervisorTest < ::Test::Unit::TestCase
   end
 
   def test_load_config_for_daemonize
-    tmp_dir = "#{TMP_DIR}/dir/test_load_config.conf"
+    tmp_dir = "#{@tmp_dir}/dir/test_load_config.conf"
     conf_info_str = %[
 <system>
   log_level info
@@ -644,7 +658,7 @@ class SupervisorTest < ::Test::Unit::TestCase
   )
   def test_logger_with_rotate_age_and_rotate_size(rotate_age)
     opts = Fluent::Supervisor.default_options.merge(
-      log_path: "#{TMP_DIR}/test", log_rotate_age: rotate_age, log_rotate_size: 10
+      log_path: "#{@tmp_dir}/test", log_rotate_age: rotate_age, log_rotate_size: 10
     )
     sv = Fluent::Supervisor.new(opts)
     log = sv.instance_variable_get(:@log)
@@ -674,7 +688,7 @@ class SupervisorTest < ::Test::Unit::TestCase
         file.puts(config)
         file.flush
         opts = Fluent::Supervisor.default_options.merge(
-          log_path: "#{TMP_DIR}/test.log", config_path: file.path
+          log_path: "#{@tmp_dir}/test.log", config_path: file.path
         )
         sv = Fluent::Supervisor.new(opts)
 
@@ -699,7 +713,7 @@ class SupervisorTest < ::Test::Unit::TestCase
         file.puts(config)
         file.flush
         opts = Fluent::Supervisor.default_options.merge(
-          log_path: "#{TMP_DIR}/test.log", config_path: file.path, config_file_type: :yaml,
+          log_path: "#{@tmp_dir}/test.log", config_path: file.path, config_file_type: :yaml,
         )
         sv = Fluent::Supervisor.new(opts)
 
@@ -773,6 +787,14 @@ class SupervisorTest < ::Test::Unit::TestCase
     end
   end
 
+  def test_per_process_path
+    path = Fluent::Supervisor::LoggerInitializer.per_process_path("C:/tmp/test.log", :supervisor, 0)
+    assert_equal(path, "C:/tmp/test-supervisor-0.log")
+
+    path = Fluent::Supervisor::LoggerInitializer.per_process_path("C:/tmp/test.log", :worker, 1)
+    assert_equal(path, "C:/tmp/test-1.log")
+  end
+
   def create_debug_dummy_logger
     dl_opts = {}
     dl_opts[:log_level] = ServerEngine::DaemonLogger::DEBUG

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.15.2
+  version: 1.15.3
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-08-22 00:00:00.000000000 Z
+date: 2022-11-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -391,10 +391,8 @@ files:
 - ".github/ISSUE_TEMPLATE/config.yml"
 - ".github/ISSUE_TEMPLATE/feature_request.yaml"
 - ".github/PULL_REQUEST_TEMPLATE.md"
-- ".github/workflows/issue-auto-closer.yml"
 - ".github/workflows/linux-test.yaml"
 - ".github/workflows/macos-test.yaml"
-- ".github/workflows/stale-actions.yml"
 - ".github/workflows/windows-test.yaml"
 - ".gitignore"
 - ".gitlab-ci.yml"
@@ -529,6 +527,7 @@ files:
 - lib/fluent/event.rb
 - lib/fluent/event_router.rb
 - lib/fluent/ext_monitor_require.rb
+- lib/fluent/file_wrapper.rb
 - lib/fluent/filter.rb
 - lib/fluent/fluent_log_event_router.rb
 - lib/fluent/formatter.rb
@@ -557,7 +556,6 @@ files:
 - lib/fluent/plugin/compressable.rb
 - lib/fluent/plugin/exec_util.rb
 - lib/fluent/plugin/file_util.rb
-- lib/fluent/plugin/file_wrapper.rb
 - lib/fluent/plugin/filter.rb
 - lib/fluent/plugin/filter_grep.rb
 - lib/fluent/plugin/filter_parser.rb
@@ -790,7 +788,6 @@ files:
 - test/plugin/test_buffer_memory_chunk.rb
 - test/plugin/test_compressable.rb
 - test/plugin/test_file_util.rb
-- test/plugin/test_file_wrapper.rb
 - test/plugin/test_filter.rb
 - test/plugin/test_filter_grep.rb
 - test/plugin/test_filter_parser.rb
@@ -922,6 +919,7 @@ files:
 - test/test_event.rb
 - test/test_event_router.rb
 - test/test_event_time.rb
+- test/test_file_wrapper.rb
 - test/test_filter.rb
 - test/test_fluent_log_event_router.rb
 - test/test_formatter.rb
@@ -1033,7 +1031,6 @@ test_files:
 - test/plugin/test_buffer_memory_chunk.rb
 - test/plugin/test_compressable.rb
 - test/plugin/test_file_util.rb
-- test/plugin/test_file_wrapper.rb
 - test/plugin/test_filter.rb
 - test/plugin/test_filter_grep.rb
 - test/plugin/test_filter_parser.rb
@@ -1165,6 +1162,7 @@ test_files:
 - test/test_event.rb
 - test/test_event_router.rb
 - test/test_event_time.rb
+- test/test_file_wrapper.rb
 - test/test_filter.rb
 - test/test_fluent_log_event_router.rb
 - test/test_formatter.rb

data/.github/workflows/issue-auto-closer.yml

@@ -1,12 +0,0 @@
-name: Autocloser
-on: [issues]
-jobs:
-  autoclose:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Autoclose issues that did not follow issue template
-      uses: roots/issue-closer-action@v1.1
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        issue-close-message: "@${issue.user.login} this issue was automatically closed because it did not follow the issue template"
-        issue-pattern: "(.*Describe the bug.*)|(.*Is your feature request related to a problem.*)"

data/.github/workflows/stale-actions.yml

@@ -1,22 +0,0 @@
-name: "Mark or close stale issues and PRs"
-on:
-  schedule:
-  - cron: "00 10 * * *"
-
-jobs:
-  stale:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/stale@v3
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        days-before-stale: 90
-        days-before-close: 30
-        stale-issue-message: "This issue has been automatically marked as stale because it has been open 90 days with no activity. Remove stale label or comment or this issue will be closed in 30 days"
-        stale-pr-message: "This PR has been automatically marked as stale because it has been open 90 days with no activity. Remove stale label or comment or this PR will be closed in 30 days"
-        close-issue-message: "This issue was automatically closed because of stale in 30 days"
-        close-pr-message: "This PR was automatically closed because of stale in 30 days"
-        stale-pr-label: "stale"
-        stale-issue-label: "stale"
-        exempt-issue-labels: "bug,enhancement,feature request,pending,work_in_progress,v1,v2"
-        exempt-pr-labels: "bug,enhancement,feature request,pending,work_in_progress,v1,v2"

data/lib/fluent/plugin/file_wrapper.rb

@@ -1,131 +0,0 @@
-#
-# Fluentd
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-
-require 'fluent/win32api'
-
-module Fluent
-  module FileWrapper
-    def self.open(path, mode='r')
-      io = WindowsFile.new(path, mode).io
-      if block_given?
-        v = yield io
-        io.close
-        v
-      else
-        io
-      end
-    end
-
-    def self.stat(path)
-      f = WindowsFile.new(path)
-      s = f.stat
-      f.close
-      s
-    end
-  end
-
-  class WindowsFile
-    include File::Constants
-
-    attr_reader :io
-
-    INVALID_HANDLE_VALUE = -1
-
-    def initialize(path, mode='r')
-      @path = path
-      @io = File.open(path, mode2flags(mode))
-      @file_handle = Win32API._get_osfhandle(@io.to_i)
-      @io.instance_variable_set(:@file_index, self.ino)
-      def @io.ino
-        @file_index
-      end
-    end
-
-    def close
-      @io.close
-      @file_handle = INVALID_HANDLE_VALUE
-    end
-
-    # To keep backward compatibility, we continue to use GetFileInformationByHandle()
-    # to get file id.
-    # Note that Ruby's File.stat uses GetFileInformationByHandleEx() with FileIdInfo
-    # and returned value is different with above one, former one is 64 bit while
-    # later one is 128bit.
-    def ino
-      by_handle_file_information = '\0'*(4+8+8+8+4+4+4+4+4+4)   #72bytes
-
-      unless Win32API.GetFileInformationByHandle(@file_handle, by_handle_file_information)
-        return 0
-      end
-
-      by_handle_file_information.unpack("I11Q1")[11] # fileindex
-    end
-
-    def stat
-      raise Errno::ENOENT if delete_pending
-      s = File.stat(@path)
-      s.instance_variable_set :@ino, self.ino
-      def s.ino; @ino; end
-      s
-    end
-
-    private
-
-    def mode2flags(mode)
-      # Always inject File::Constants::SHARE_DELETE
-      # https://github.com/fluent/fluentd/pull/3585#issuecomment-1101502617
-      # To enable SHARE_DELETE, BINARY is also required.
-      # https://bugs.ruby-lang.org/issues/11218
-      # https://github.com/ruby/ruby/blob/d6684f063bc53e3cab025bd39526eca3b480b5e7/win32/win32.c#L6332-L6345
-      flags = BINARY | SHARE_DELETE
-      case mode.delete("b")
-      when "r"
-        flags |= RDONLY
-      when "r+"
-        flags |= RDWR
-      when "w"
-        flags |= WRONLY | CREAT | TRUNC
-      when "w+"
-        flags |= RDWR | CREAT | TRUNC
-      when "a"
-        flags |= WRONLY | CREAT | APPEND
-      when "a+"
-        flags |= RDWR | CREAT | APPEND
-      else
-        raise Errno::EINVAL.new("Unsupported mode by Fluent::FileWrapper: #{mode}")
-      end
-    end
-
-    # DeletePending is a Windows-specific file state that roughly means
-    # "this file is queued for deletion, so close any open handlers"
-    #
-    # This flag can be retrieved via GetFileInformationByHandleEx().
-    #
-    # https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-getfileinformationbyhandleex
-    #
-    def delete_pending
-      file_standard_info = 0x01
-      bufsize = 1024
-      buf = '\0' * bufsize
-
-      unless Win32API.GetFileInformationByHandleEx(@file_handle, file_standard_info, buf, bufsize)
-        return false
-      end
-
-      return buf.unpack("QQICC")[3] != 0
-    end
-  end
-end if Fluent.windows?