fluentd 1.15.2 → 1.15.3

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: af4e72332c4c1128c66e129579a50d0edd10b2bbeae16d805ed57f13cf425ae2
4
- data.tar.gz: de907aa30f8d0352a871b8fb6cc938ffdffef7012910dbb99cc3986b69b6929e
3
+ metadata.gz: 0e7d3798a39136da859e23e8bee742d8db252f58ce410cc530710d60dba36c05
4
+ data.tar.gz: 82146c03770da1ebbad670fbd356a3020846f27d72e1eaeaed9538fc0b99c9cb
5
5
  SHA512:
6
- metadata.gz: 908bc3de0f3d959d520e91cd62e6191f9b16c333769b8d1965fd7d0a8bdb585ad203d86fd5061b5d0e45cf7db33e81c2f44dd3f45cb81ee6c28dc287838f349f
7
- data.tar.gz: c3e4b97d24ffb3f699d6d0d8003ae8bdcb1b83b2d0c983ae725f98024935cbc39742016efdbd106387cd8b0880dcc57ca837bde0820a568a301a8937337638b6
6
+ metadata.gz: 4b895809bf5e79c5fea8e5a77a485129d8a5912327fb7cbfc176a85e77918068cf24db37cf6e8fba53f24bbdb7e6af7d593a621bcad575faf4299589c8cb662c
7
+ data.tar.gz: a0985cfd48858c88f7ec9feb5c87e035361998ae50be98d6f486d9ae897cce350cf1878cd4a2491c40c6faa5a1cb67b3c52484d1100694bb624894635b8bebd6
data/CHANGELOG.md CHANGED
@@ -1,5 +1,24 @@
1
1
  # v1.15
2
2
 
3
+ ## Release v1.15.3 - 2022/11/02
4
+
5
+ ### Bug Fix
6
+
7
+ * Support glob for `!include` directive in YAML config format
8
+ https://github.com/fluent/fluentd/pull/3917
9
+ * Remove meaningless oj options
10
+ https://github.com/fluent/fluentd/pull/3929
11
+ * Fix log initializer to correctly create per-process files on Windows
12
+ https://github.com/fluent/fluentd/pull/3939
13
+ * out_file: Fix the multi-worker check with `<worker 0-N>` directive
14
+ https://github.com/fluent/fluentd/pull/3942
15
+
16
+ ### Misc
17
+
18
+ * Fix broken tests on Ruby 3.2
19
+ https://github.com/fluent/fluentd/pull/3883
20
+ https://github.com/fluent/fluentd/pull/3922
21
+
3
22
  ## Release v1.15.2 - 2022/08/22
4
23
 
5
24
  ### Enhancement
@@ -345,17 +345,6 @@ end
345
345
  exit 0 if early_exit
346
346
 
347
347
  if opts[:supervise]
348
- if Fluent.windows?
349
- if opts[:log_path] && opts[:log_path] != "-"
350
- if opts[:log_rotate_age] || opts[:log_rotate_size]
351
- require 'pathname'
352
-
353
- log_path = Pathname(opts[:log_path]).sub_ext("-supervisor#{Pathname(opts[:log_path]).extname}").to_s
354
- opts[:log_path] = log_path
355
- end
356
- end
357
- end
358
-
359
348
  supervisor = Fluent::Supervisor.new(opts)
360
349
  supervisor.configure(supervisor: true)
361
350
  supervisor.run_supervisor(dry_run: opts[:dry_run])
@@ -44,7 +44,7 @@ module Fluent
44
44
  visitor = Visitor.new(scanner, class_loader)
45
45
 
46
46
  visitor._register_domain(INCLUDE_TAG) do |_, val|
47
- load(path.parent.join(val))
47
+ eval_include(Pathname.new(val), path.parent)
48
48
  end
49
49
 
50
50
  visitor._register_domain(FLUENT_JSON_TAG) do |_, val|
@@ -60,6 +60,23 @@ module Fluent
60
60
  end
61
61
  end
62
62
 
63
+ def eval_include(path, parent)
64
+ if path.relative?
65
+ pattern = parent.join(path)
66
+ else
67
+ pattern = path
68
+ end
69
+ result = []
70
+ Dir.glob(pattern).sort.each do |path|
71
+ result.concat(load(Pathname.new(path)))
72
+ end
73
+ result
74
+ rescue SystemCallError => e
75
+ parse_error = ConfigParseError.new("include error #{path} - #{e}")
76
+ parse_error.set_backtrace(e.backtrace)
77
+ raise parse_error
78
+ end
79
+
63
80
  class Visitor < Psych::Visitors::ToRuby
64
81
  def initialize(scanner, class_loader)
65
82
  super(scanner, class_loader)
@@ -0,0 +1,137 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ unless Fluent.windows?
18
+ Fluent::FileWrapper = File
19
+ else
20
+ require 'fluent/win32api'
21
+
22
+ module Fluent
23
+ module FileWrapper
24
+ def self.open(path, mode='r')
25
+ io = WindowsFile.new(path, mode).io
26
+ if block_given?
27
+ v = yield io
28
+ io.close
29
+ v
30
+ else
31
+ io
32
+ end
33
+ end
34
+
35
+ def self.stat(path)
36
+ f = WindowsFile.new(path)
37
+ s = f.stat
38
+ f.close
39
+ s
40
+ end
41
+ end
42
+
43
+ class WindowsFile
44
+ include File::Constants
45
+
46
+ attr_reader :io
47
+
48
+ INVALID_HANDLE_VALUE = -1
49
+
50
+ def initialize(path, mode_enc='r')
51
+ @path = path
52
+ mode, enc = mode_enc.split(":", 2)
53
+ @io = File.open(path, mode2flags(mode))
54
+ @io.set_encoding(enc) if enc
55
+ @file_handle = Win32API._get_osfhandle(@io.to_i)
56
+ @io.instance_variable_set(:@file_index, self.ino)
57
+ def @io.ino
58
+ @file_index
59
+ end
60
+ end
61
+
62
+ def close
63
+ @io.close
64
+ @file_handle = INVALID_HANDLE_VALUE
65
+ end
66
+
67
+ # To keep backward compatibility, we continue to use GetFileInformationByHandle()
68
+ # to get file id.
69
+ # Note that Ruby's File.stat uses GetFileInformationByHandleEx() with FileIdInfo
70
+ # and returned value is different with above one, former one is 64 bit while
71
+ # later one is 128bit.
72
+ def ino
73
+ by_handle_file_information = '\0'*(4+8+8+8+4+4+4+4+4+4) #72bytes
74
+
75
+ unless Win32API.GetFileInformationByHandle(@file_handle, by_handle_file_information)
76
+ return 0
77
+ end
78
+
79
+ by_handle_file_information.unpack("I11Q1")[11] # fileindex
80
+ end
81
+
82
+ def stat
83
+ raise Errno::ENOENT if delete_pending
84
+ s = File.stat(@path)
85
+ s.instance_variable_set :@ino, self.ino
86
+ def s.ino; @ino; end
87
+ s
88
+ end
89
+
90
+ private
91
+
92
+ def mode2flags(mode)
93
+ # Always inject File::Constants::SHARE_DELETE
94
+ # https://github.com/fluent/fluentd/pull/3585#issuecomment-1101502617
95
+ # To enable SHARE_DELETE, BINARY is also required.
96
+ # https://bugs.ruby-lang.org/issues/11218
97
+ # https://github.com/ruby/ruby/blob/d6684f063bc53e3cab025bd39526eca3b480b5e7/win32/win32.c#L6332-L6345
98
+ flags = BINARY | SHARE_DELETE
99
+ case mode.delete("b")
100
+ when "r"
101
+ flags |= RDONLY
102
+ when "r+"
103
+ flags |= RDWR
104
+ when "w"
105
+ flags |= WRONLY | CREAT | TRUNC
106
+ when "w+"
107
+ flags |= RDWR | CREAT | TRUNC
108
+ when "a"
109
+ flags |= WRONLY | CREAT | APPEND
110
+ when "a+"
111
+ flags |= RDWR | CREAT | APPEND
112
+ else
113
+ raise Errno::EINVAL.new("Unsupported mode by Fluent::FileWrapper: #{mode}")
114
+ end
115
+ end
116
+
117
+ # DeletePending is a Windows-specific file state that roughly means
118
+ # "this file is queued for deletion, so close any open handlers"
119
+ #
120
+ # This flag can be retrieved via GetFileInformationByHandleEx().
121
+ #
122
+ # https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-getfileinformationbyhandleex
123
+ #
124
+ def delete_pending
125
+ file_standard_info = 0x01
126
+ bufsize = 1024
127
+ buf = '\0' * bufsize
128
+
129
+ unless Win32API.GetFileInformationByHandleEx(@file_handle, file_standard_info, buf, bufsize)
130
+ return false
131
+ end
132
+
133
+ return buf.unpack("QQICC")[3] != 0
134
+ end
135
+ end
136
+ end
137
+ end
@@ -4,14 +4,13 @@ module Fluent
4
4
  class OjOptions
5
5
  OPTIONS = {
6
6
  'bigdecimal_load': :symbol,
7
- 'max_nesting': :integer,
8
7
  'mode': :symbol,
9
8
  'use_to_json': :bool
10
9
  }
11
10
 
12
11
  ALLOWED_VALUES = {
13
12
  'bigdecimal_load': %i[bigdecimal float auto],
14
- 'mode': %i[strict null compat json rails object custom]
13
+ 'mode': %i[strict null compat json rails custom]
15
14
  }
16
15
 
17
16
  DEFAULTS = {
@@ -25,12 +25,7 @@ require 'fluent/variable_store'
25
25
  require 'fluent/capability'
26
26
  require 'fluent/plugin/in_tail/position_file'
27
27
  require 'fluent/plugin/in_tail/group_watch'
28
-
29
- if Fluent.windows?
30
- require_relative 'file_wrapper'
31
- else
32
- Fluent::FileWrapper = File
33
- end
28
+ require 'fluent/file_wrapper'
34
29
 
35
30
  module Fluent::Plugin
36
31
  class TailInput < Fluent::Plugin::Input
@@ -188,10 +188,6 @@ module Fluent::Plugin
188
188
  condition = Gem::Dependency.new('', [">= 2.7.0", "< 3.1.0"])
189
189
  @need_ruby_on_macos_workaround = true if condition.match?('', RUBY_VERSION)
190
190
  end
191
-
192
- if @need_lock && @append && @fluentd_lock_dir.nil?
193
- raise Fluent::InvalidLockDirectory, "must set FLUENTD_LOCK_DIR on multi-worker append mode"
194
- end
195
191
  end
196
192
 
197
193
  def multi_workers_ready?
@@ -525,10 +525,22 @@ module Fluent
525
525
  @log_rotate_size = log_rotate_size
526
526
  end
527
527
 
528
- def worker_id_suffixed_path(worker_id, path)
529
- require 'pathname'
530
-
531
- Pathname(path).sub_ext("-#{worker_id}#{Pathname(path).extname}").to_s
528
+ # Create a unique path for each process.
529
+ #
530
+ # >>> per_process_path(:worker, 1, "C:/tmp/test.log")
531
+ # C:/tmp/test-1.log
532
+ # >>> per_process_path(:supervisor, 0, "C:/tmp/test.log")
533
+ # C:/tmp/test-supervisor-0.log
534
+ def self.per_process_path(path, process_type, worker_id)
535
+ path = Pathname(path)
536
+ ext = path.extname
537
+
538
+ if process_type == :supervisor
539
+ suffix = "-#{process_type}-0#{ext}" # "-0" for backword compatibility.
540
+ else
541
+ suffix = "-#{worker_id}#{ext}"
542
+ end
543
+ return path.sub_ext(suffix).to_s
532
544
  end
533
545
 
534
546
  def init(process_type, worker_id)
@@ -540,13 +552,19 @@ module Fluent
540
552
  FileUtils.mkdir_p(File.dirname(@path))
541
553
  end
542
554
 
543
- @logdev = if @log_rotate_age || @log_rotate_size
544
- Fluent::LogDeviceIO.new(Fluent.windows? ?
545
- worker_id_suffixed_path(worker_id, @path) : @path,
546
- shift_age: @log_rotate_age, shift_size: @log_rotate_size)
547
- else
548
- File.open(@path, "a")
549
- end
555
+ if @log_rotate_age || @log_rotate_size
556
+ # We need to prepare a unique path for each worker since
557
+ # Windows locks files.
558
+ if Fluent.windows?
559
+ path = LoggerInitializer.per_process_path(@path, process_type, worker_id)
560
+ else
561
+ path = @path
562
+ end
563
+ @logdev = Fluent::LogDeviceIO.new(path, shift_age: @log_rotate_age, shift_size: @log_rotate_size)
564
+ else
565
+ @logdev = File.open(@path, "a")
566
+ end
567
+
550
568
  if @chuser || @chgroup
551
569
  chuid = @chuser ? ServerEngine::Privilege.get_etc_passwd(@chuser).uid : nil
552
570
  chgid = @chgroup ? ServerEngine::Privilege.get_etc_group(@chgroup).gid : nil
@@ -565,6 +583,7 @@ module Fluent
565
583
  $log = Fluent::Log.new(logger, @opts)
566
584
  $log.enable_color(false) if @path
567
585
  $log.enable_debug if @level <= Fluent::Log::LEVEL_DEBUG
586
+ $log.info "init #{process_type} logger", path: path, rotate_age: @log_rotate_age, rotate_size: @log_rotate_size
568
587
  end
569
588
 
570
589
  def stdout?
@@ -16,6 +16,6 @@
16
16
 
17
17
  module Fluent
18
18
 
19
- VERSION = '1.15.2'
19
+ VERSION = '1.15.3'
20
20
 
21
21
  end
@@ -5,20 +5,35 @@ require_relative '../helper'
5
5
 
6
6
  require 'fileutils'
7
7
  require 'timeout'
8
+ require 'securerandom'
9
+ require 'fluent/file_wrapper'
8
10
 
9
11
  class TestFluentdCommand < ::Test::Unit::TestCase
10
- TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/command/fluentd#{ENV['TEST_ENV_NUMBER']}")
11
12
  SUPERVISOR_PID_PATTERN = /starting fluentd-[.0-9]+ pid=(\d+)/
12
13
  WORKER_PID_PATTERN = /starting fluentd worker pid=(\d+) /
13
14
 
15
+ def tmp_dir
16
+ File.join(File.dirname(__FILE__), "..", "tmp", "command" "fluentd#{ENV['TEST_ENV_NUMBER']}", SecureRandom.hex(10))
17
+ end
18
+
14
19
  setup do
15
- FileUtils.rm_rf(TMP_DIR)
16
- FileUtils.mkdir_p(TMP_DIR)
20
+ @tmp_dir = tmp_dir
21
+ FileUtils.mkdir_p(@tmp_dir)
17
22
  @supervisor_pid = nil
18
23
  @worker_pids = []
19
24
  ENV["TEST_RUBY_PATH"] = nil
20
25
  end
21
26
 
27
+ teardown do
28
+ begin
29
+ FileUtils.rm_rf(@tmp_dir)
30
+ rescue Errno::EACCES
31
+ # It may occur on Windows because of delete pending state due to delayed GC.
32
+ # Ruby 3.2 or later doesn't ignore Errno::EACCES:
33
+ # https://github.com/ruby/ruby/commit/983115cf3c8f75b1afbe3274f02c1529e1ce3a81
34
+ end
35
+ end
36
+
22
37
  def process_exist?(pid)
23
38
  begin
24
39
  r = Process.waitpid(pid, Process::WNOHANG)
@@ -30,17 +45,17 @@ class TestFluentdCommand < ::Test::Unit::TestCase
30
45
  end
31
46
 
32
47
  def create_conf_file(name, content, ext_enc = 'utf-8')
33
- conf_path = File.join(TMP_DIR, name)
34
- File.open(conf_path, "w:#{ext_enc}:utf-8") do |file|
48
+ conf_path = File.join(@tmp_dir, name)
49
+ Fluent::FileWrapper.open(conf_path, "w:#{ext_enc}:utf-8") do |file|
35
50
  file.write content
36
51
  end
37
52
  conf_path
38
53
  end
39
54
 
40
55
  def create_plugin_file(name, content)
41
- file_path = File.join(TMP_DIR, 'plugin', name)
56
+ file_path = File.join(@tmp_dir, 'plugin', name)
42
57
  FileUtils.mkdir_p(File.dirname(file_path))
43
- File.open(file_path, 'w') do |file|
58
+ Fluent::FileWrapper.open(file_path, 'w') do |file|
44
59
  file.write content
45
60
  end
46
61
  file_path
@@ -56,8 +71,8 @@ class TestFluentdCommand < ::Test::Unit::TestCase
56
71
  end
57
72
  end
58
73
 
59
- def execute_command(cmdline, chdir=TMP_DIR, env = {})
60
- null_stream = File.open(File::NULL, 'w')
74
+ def execute_command(cmdline, chdir=@tmp_dir, env = {})
75
+ null_stream = Fluent::FileWrapper.open(File::NULL, 'w')
61
76
  gemfile_path = File.expand_path(File.dirname(__FILE__) + "../../../Gemfile")
62
77
 
63
78
  env = { "BUNDLE_GEMFILE" => gemfile_path }.merge(env)
@@ -103,7 +118,7 @@ class TestFluentdCommand < ::Test::Unit::TestCase
103
118
  assert_error_msg = ""
104
119
  stdio_buf = ""
105
120
  begin
106
- execute_command(cmdline, TMP_DIR, env) do |pid, stdout|
121
+ execute_command(cmdline, @tmp_dir, env) do |pid, stdout|
107
122
  begin
108
123
  waiting(timeout) do
109
124
  while process_exist?(pid) && !matched
@@ -269,7 +284,7 @@ CONF
269
284
 
270
285
  sub_test_case 'with system configuration about root directory' do
271
286
  setup do
272
- @root_path = File.join(TMP_DIR, "rootpath")
287
+ @root_path = File.join(@tmp_dir, "rootpath")
273
288
  FileUtils.rm_rf(@root_path)
274
289
  @conf = <<CONF
275
290
  <system>
@@ -308,7 +323,7 @@ CONF
308
323
  end
309
324
 
310
325
  test 'fails to launch fluentd if specified root path is invalid path for directory' do
311
- File.open(@root_path, 'w') do |_|
326
+ Fluent::FileWrapper.open(@root_path, 'w') do |_|
312
327
  # create file and close it
313
328
  end
314
329
  conf_path = create_conf_file('existing_root_dir.conf', @conf)
@@ -508,7 +523,7 @@ CONF
508
523
 
509
524
  assert_fluentd_fails_to_start(
510
525
  create_cmdline(conf_path, "-p", File.dirname(plugin_path)),
511
- "in_buggy.rb:5: syntax error, unexpected end-of-input, expecting"
526
+ "in_buggy.rb:5: syntax error, unexpected end-of-input"
512
527
  )
513
528
  end
514
529
  end
@@ -554,7 +569,7 @@ CONF
554
569
 
555
570
  sub_test_case 'configured to run 2 workers' do
556
571
  setup do
557
- @root_path = File.join(TMP_DIR, "rootpath")
572
+ @root_path = File.join(@tmp_dir, "rootpath")
558
573
  FileUtils.rm_rf(@root_path)
559
574
  FileUtils.mkdir_p(@root_path)
560
575
  end
@@ -961,10 +976,10 @@ CONF
961
976
  </match>
962
977
  CONF
963
978
  ruby_path = ServerEngine.ruby_bin_path
964
- tmp_ruby_path = File.join(TMP_DIR, "ruby with spaces")
979
+ tmp_ruby_path = File.join(@tmp_dir, "ruby with spaces")
965
980
  if Fluent.windows?
966
981
  tmp_ruby_path << ".bat"
967
- File.open(tmp_ruby_path, "w") do |file|
982
+ Fluent::FileWrapper.open(tmp_ruby_path, "w") do |file|
968
983
  file.write "#{ruby_path} %*"
969
984
  end
970
985
  else