fluentd 1.13.3 → 1.16.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (179) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/{bug_report.yaml → bug_report.yml} +2 -0
  3. data/.github/ISSUE_TEMPLATE/config.yml +2 -2
  4. data/.github/ISSUE_TEMPLATE/{feature_request.yaml → feature_request.yml} +1 -0
  5. data/.github/workflows/stale-actions.yml +11 -9
  6. data/.github/workflows/test.yml +32 -0
  7. data/CHANGELOG.md +490 -10
  8. data/CONTRIBUTING.md +2 -2
  9. data/MAINTAINERS.md +7 -5
  10. data/README.md +3 -23
  11. data/Rakefile +1 -1
  12. data/SECURITY.md +14 -0
  13. data/fluentd.gemspec +7 -8
  14. data/lib/fluent/command/cat.rb +13 -3
  15. data/lib/fluent/command/ctl.rb +6 -3
  16. data/lib/fluent/command/fluentd.rb +73 -65
  17. data/lib/fluent/command/plugin_config_formatter.rb +1 -1
  18. data/lib/fluent/compat/output.rb +9 -6
  19. data/lib/fluent/config/dsl.rb +1 -1
  20. data/lib/fluent/config/error.rb +12 -0
  21. data/lib/fluent/config/literal_parser.rb +2 -2
  22. data/lib/fluent/config/parser.rb +1 -1
  23. data/lib/fluent/config/v1_parser.rb +3 -3
  24. data/lib/fluent/config/yaml_parser/fluent_value.rb +47 -0
  25. data/lib/fluent/config/yaml_parser/loader.rb +108 -0
  26. data/lib/fluent/config/yaml_parser/parser.rb +166 -0
  27. data/lib/fluent/config/yaml_parser/section_builder.rb +107 -0
  28. data/lib/fluent/config/yaml_parser.rb +56 -0
  29. data/lib/fluent/config.rb +14 -1
  30. data/lib/fluent/counter/server.rb +1 -1
  31. data/lib/fluent/counter/validator.rb +3 -3
  32. data/lib/fluent/daemon.rb +2 -4
  33. data/lib/fluent/engine.rb +1 -1
  34. data/lib/fluent/env.rb +4 -0
  35. data/lib/fluent/error.rb +3 -0
  36. data/lib/fluent/event.rb +8 -4
  37. data/lib/fluent/event_router.rb +47 -2
  38. data/lib/fluent/file_wrapper.rb +137 -0
  39. data/lib/fluent/log/console_adapter.rb +66 -0
  40. data/lib/fluent/log.rb +44 -5
  41. data/lib/fluent/match.rb +1 -1
  42. data/lib/fluent/msgpack_factory.rb +6 -1
  43. data/lib/fluent/oj_options.rb +1 -2
  44. data/lib/fluent/plugin/bare_output.rb +49 -8
  45. data/lib/fluent/plugin/base.rb +26 -9
  46. data/lib/fluent/plugin/buf_file.rb +34 -5
  47. data/lib/fluent/plugin/buf_file_single.rb +32 -3
  48. data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
  49. data/lib/fluent/plugin/buffer.rb +216 -70
  50. data/lib/fluent/plugin/filter.rb +35 -1
  51. data/lib/fluent/plugin/filter_record_transformer.rb +1 -1
  52. data/lib/fluent/plugin/in_forward.rb +2 -2
  53. data/lib/fluent/plugin/in_http.rb +39 -10
  54. data/lib/fluent/plugin/in_monitor_agent.rb +4 -2
  55. data/lib/fluent/plugin/in_sample.rb +1 -1
  56. data/lib/fluent/plugin/in_syslog.rb +13 -1
  57. data/lib/fluent/plugin/in_tail/group_watch.rb +204 -0
  58. data/lib/fluent/plugin/in_tail/position_file.rb +33 -33
  59. data/lib/fluent/plugin/in_tail.rb +216 -84
  60. data/lib/fluent/plugin/in_tcp.rb +47 -2
  61. data/lib/fluent/plugin/input.rb +39 -1
  62. data/lib/fluent/plugin/metrics.rb +119 -0
  63. data/lib/fluent/plugin/metrics_local.rb +96 -0
  64. data/lib/fluent/plugin/multi_output.rb +43 -6
  65. data/lib/fluent/plugin/out_copy.rb +1 -1
  66. data/lib/fluent/plugin/out_exec_filter.rb +2 -2
  67. data/lib/fluent/plugin/out_file.rb +20 -2
  68. data/lib/fluent/plugin/out_forward/ack_handler.rb +19 -4
  69. data/lib/fluent/plugin/out_forward/socket_cache.rb +2 -0
  70. data/lib/fluent/plugin/out_forward.rb +17 -9
  71. data/lib/fluent/plugin/out_secondary_file.rb +39 -22
  72. data/lib/fluent/plugin/output.rb +167 -78
  73. data/lib/fluent/plugin/parser.rb +3 -4
  74. data/lib/fluent/plugin/parser_apache2.rb +1 -1
  75. data/lib/fluent/plugin/parser_json.rb +1 -1
  76. data/lib/fluent/plugin/parser_syslog.rb +1 -1
  77. data/lib/fluent/plugin/storage_local.rb +3 -5
  78. data/lib/fluent/plugin.rb +10 -1
  79. data/lib/fluent/plugin_helper/child_process.rb +3 -0
  80. data/lib/fluent/plugin_helper/event_emitter.rb +8 -1
  81. data/lib/fluent/plugin_helper/event_loop.rb +2 -2
  82. data/lib/fluent/plugin_helper/http_server/server.rb +2 -1
  83. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  84. data/lib/fluent/plugin_helper/record_accessor.rb +1 -1
  85. data/lib/fluent/plugin_helper/retry_state.rb +14 -4
  86. data/lib/fluent/plugin_helper/server.rb +35 -6
  87. data/lib/fluent/plugin_helper/service_discovery.rb +2 -2
  88. data/lib/fluent/plugin_helper/socket.rb +13 -2
  89. data/lib/fluent/plugin_helper/thread.rb +3 -3
  90. data/lib/fluent/plugin_helper.rb +1 -0
  91. data/lib/fluent/plugin_id.rb +3 -2
  92. data/lib/fluent/registry.rb +2 -1
  93. data/lib/fluent/root_agent.rb +6 -0
  94. data/lib/fluent/rpc.rb +4 -3
  95. data/lib/fluent/supervisor.rb +283 -259
  96. data/lib/fluent/system_config.rb +13 -3
  97. data/lib/fluent/test/driver/base.rb +11 -5
  98. data/lib/fluent/test/driver/filter.rb +4 -0
  99. data/lib/fluent/test/startup_shutdown.rb +6 -8
  100. data/lib/fluent/time.rb +21 -20
  101. data/lib/fluent/version.rb +1 -1
  102. data/lib/fluent/win32api.rb +38 -0
  103. data/lib/fluent/winsvc.rb +5 -8
  104. data/templates/new_gem/test/helper.rb.erb +0 -1
  105. data/test/command/test_cat.rb +31 -2
  106. data/test/command/test_ctl.rb +1 -2
  107. data/test/command/test_fluentd.rb +209 -24
  108. data/test/command/test_plugin_config_formatter.rb +0 -1
  109. data/test/compat/test_parser.rb +6 -6
  110. data/test/config/test_system_config.rb +13 -11
  111. data/test/config/test_types.rb +1 -1
  112. data/test/log/test_console_adapter.rb +110 -0
  113. data/test/plugin/in_tail/test_io_handler.rb +26 -8
  114. data/test/plugin/in_tail/test_position_file.rb +48 -59
  115. data/test/plugin/out_forward/test_ack_handler.rb +39 -0
  116. data/test/plugin/out_forward/test_socket_cache.rb +26 -1
  117. data/test/plugin/test_bare_output.rb +14 -1
  118. data/test/plugin/test_base.rb +133 -1
  119. data/test/plugin/test_buf_file.rb +62 -23
  120. data/test/plugin/test_buf_file_single.rb +65 -0
  121. data/test/plugin/test_buffer.rb +267 -3
  122. data/test/plugin/test_buffer_chunk.rb +11 -0
  123. data/test/plugin/test_filter.rb +12 -1
  124. data/test/plugin/test_filter_parser.rb +1 -1
  125. data/test/plugin/test_filter_stdout.rb +2 -2
  126. data/test/plugin/test_in_forward.rb +9 -11
  127. data/test/plugin/test_in_http.rb +65 -3
  128. data/test/plugin/test_in_monitor_agent.rb +216 -11
  129. data/test/plugin/test_in_object_space.rb +9 -3
  130. data/test/plugin/test_in_syslog.rb +35 -0
  131. data/test/plugin/test_in_tail.rb +1393 -385
  132. data/test/plugin/test_in_tcp.rb +87 -2
  133. data/test/plugin/test_in_udp.rb +28 -0
  134. data/test/plugin/test_in_unix.rb +2 -2
  135. data/test/plugin/test_input.rb +12 -1
  136. data/test/plugin/test_metrics.rb +294 -0
  137. data/test/plugin/test_metrics_local.rb +96 -0
  138. data/test/plugin/test_multi_output.rb +25 -1
  139. data/test/plugin/test_out_exec.rb +6 -4
  140. data/test/plugin/test_out_exec_filter.rb +6 -2
  141. data/test/plugin/test_out_file.rb +34 -17
  142. data/test/plugin/test_out_forward.rb +78 -77
  143. data/test/plugin/test_out_http.rb +1 -0
  144. data/test/plugin/test_out_stdout.rb +2 -2
  145. data/test/plugin/test_output.rb +297 -12
  146. data/test/plugin/test_output_as_buffered.rb +44 -44
  147. data/test/plugin/test_output_as_buffered_compress.rb +32 -18
  148. data/test/plugin/test_output_as_buffered_retries.rb +54 -7
  149. data/test/plugin/test_output_as_buffered_secondary.rb +4 -4
  150. data/test/plugin/test_parser_regexp.rb +1 -6
  151. data/test/plugin/test_parser_syslog.rb +1 -1
  152. data/test/plugin_helper/test_cert_option.rb +1 -1
  153. data/test/plugin_helper/test_child_process.rb +38 -16
  154. data/test/plugin_helper/test_event_emitter.rb +29 -0
  155. data/test/plugin_helper/test_http_server_helper.rb +1 -1
  156. data/test/plugin_helper/test_metrics.rb +137 -0
  157. data/test/plugin_helper/test_retry_state.rb +602 -38
  158. data/test/plugin_helper/test_server.rb +78 -6
  159. data/test/plugin_helper/test_timer.rb +2 -2
  160. data/test/test_config.rb +191 -24
  161. data/test/test_event_router.rb +17 -0
  162. data/test/test_file_wrapper.rb +53 -0
  163. data/test/test_formatter.rb +24 -21
  164. data/test/test_log.rb +122 -40
  165. data/test/test_msgpack_factory.rb +32 -0
  166. data/test/test_plugin_classes.rb +102 -0
  167. data/test/test_root_agent.rb +30 -1
  168. data/test/test_supervisor.rb +477 -257
  169. data/test/test_time_parser.rb +22 -0
  170. metadata +55 -34
  171. data/.drone.yml +0 -35
  172. data/.github/workflows/issue-auto-closer.yml +0 -12
  173. data/.github/workflows/linux-test.yaml +0 -36
  174. data/.github/workflows/macos-test.yaml +0 -30
  175. data/.github/workflows/windows-test.yaml +0 -46
  176. data/.gitlab-ci.yml +0 -103
  177. data/lib/fluent/plugin/file_wrapper.rb +0 -187
  178. data/test/plugin/test_file_wrapper.rb +0 -126
  179. data/test/test_logger_initializer.rb +0 -46
@@ -5,20 +5,35 @@ require_relative '../helper'
5
5
 
6
6
  require 'fileutils'
7
7
  require 'timeout'
8
+ require 'securerandom'
9
+ require 'fluent/file_wrapper'
8
10
 
9
11
  class TestFluentdCommand < ::Test::Unit::TestCase
10
- TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/command/fluentd#{ENV['TEST_ENV_NUMBER']}")
11
12
  SUPERVISOR_PID_PATTERN = /starting fluentd-[.0-9]+ pid=(\d+)/
12
13
  WORKER_PID_PATTERN = /starting fluentd worker pid=(\d+) /
13
14
 
15
+ def tmp_dir
16
+ File.join(File.dirname(__FILE__), "..", "tmp", "command" "fluentd#{ENV['TEST_ENV_NUMBER']}", SecureRandom.hex(10))
17
+ end
18
+
14
19
  setup do
15
- FileUtils.rm_rf(TMP_DIR)
16
- FileUtils.mkdir_p(TMP_DIR)
20
+ @tmp_dir = tmp_dir
21
+ FileUtils.mkdir_p(@tmp_dir)
17
22
  @supervisor_pid = nil
18
23
  @worker_pids = []
19
24
  ENV["TEST_RUBY_PATH"] = nil
20
25
  end
21
26
 
27
+ teardown do
28
+ begin
29
+ FileUtils.rm_rf(@tmp_dir)
30
+ rescue Errno::EACCES
31
+ # It may occur on Windows because of delete pending state due to delayed GC.
32
+ # Ruby 3.2 or later doesn't ignore Errno::EACCES:
33
+ # https://github.com/ruby/ruby/commit/983115cf3c8f75b1afbe3274f02c1529e1ce3a81
34
+ end
35
+ end
36
+
22
37
  def process_exist?(pid)
23
38
  begin
24
39
  r = Process.waitpid(pid, Process::WNOHANG)
@@ -30,17 +45,17 @@ class TestFluentdCommand < ::Test::Unit::TestCase
30
45
  end
31
46
 
32
47
  def create_conf_file(name, content, ext_enc = 'utf-8')
33
- conf_path = File.join(TMP_DIR, name)
34
- File.open(conf_path, "w:#{ext_enc}:utf-8") do |file|
48
+ conf_path = File.join(@tmp_dir, name)
49
+ Fluent::FileWrapper.open(conf_path, "w:#{ext_enc}:utf-8") do |file|
35
50
  file.write content
36
51
  end
37
52
  conf_path
38
53
  end
39
54
 
40
55
  def create_plugin_file(name, content)
41
- file_path = File.join(TMP_DIR, 'plugin', name)
56
+ file_path = File.join(@tmp_dir, 'plugin', name)
42
57
  FileUtils.mkdir_p(File.dirname(file_path))
43
- File.open(file_path, 'w') do |file|
58
+ Fluent::FileWrapper.open(file_path, 'w') do |file|
44
59
  file.write content
45
60
  end
46
61
  file_path
@@ -56,8 +71,22 @@ class TestFluentdCommand < ::Test::Unit::TestCase
56
71
  end
57
72
  end
58
73
 
59
- def execute_command(cmdline, chdir=TMP_DIR, env = {})
60
- null_stream = File.open(File::NULL, 'w')
74
+ def process_kill(pid)
75
+ if Fluent.windows?
76
+ Process.kill(:KILL, pid) rescue nil
77
+ return
78
+ end
79
+
80
+ begin
81
+ Process.kill(:TERM, pid) rescue nil
82
+ Timeout.timeout(10){ sleep 0.1 while process_exist?(pid) }
83
+ rescue Timeout::Error
84
+ Process.kill(:KILL, pid) rescue nil
85
+ end
86
+ end
87
+
88
+ def execute_command(cmdline, chdir=@tmp_dir, env = {})
89
+ null_stream = Fluent::FileWrapper.open(File::NULL, 'w')
61
90
  gemfile_path = File.expand_path(File.dirname(__FILE__) + "../../../Gemfile")
62
91
 
63
92
  env = { "BUNDLE_GEMFILE" => gemfile_path }.merge(env)
@@ -70,12 +99,12 @@ class TestFluentdCommand < ::Test::Unit::TestCase
70
99
  yield pid, io
71
100
  # p(here: "execute command", pid: pid, worker_pids: @worker_pids)
72
101
  ensure
73
- Process.kill(:KILL, pid) rescue nil
102
+ process_kill(pid)
74
103
  if @supervisor_pid
75
- Process.kill(:KILL, @supervisor_pid) rescue nil
104
+ process_kill(@supervisor_pid)
76
105
  end
77
106
  @worker_pids.each do |cpid|
78
- Process.kill(:KILL, cpid) rescue nil
107
+ process_kill(cpid)
79
108
  end
80
109
  # p(here: "execute command", pid: pid, exist: process_exist?(pid), worker_pids: @worker_pids, exists: @worker_pids.map{|i| process_exist?(i) })
81
110
  Timeout.timeout(10){ sleep 0.1 while process_exist?(pid) }
@@ -97,16 +126,18 @@ class TestFluentdCommand < ::Test::Unit::TestCase
97
126
  end
98
127
  end
99
128
 
100
- def assert_log_matches(cmdline, *pattern_list, patterns_not_match: [], timeout: 10, env: {})
129
+ # ATTENTION: This stops taking logs when all `pattern_list` match or timeout,
130
+ # so `patterns_not_match` can test only logs up to that point.
131
+ def assert_log_matches(cmdline, *pattern_list, patterns_not_match: [], timeout: 20, env: {})
101
132
  matched = false
102
133
  matched_wrongly = false
103
134
  assert_error_msg = ""
104
135
  stdio_buf = ""
105
136
  begin
106
- execute_command(cmdline, TMP_DIR, env) do |pid, stdout|
137
+ execute_command(cmdline, @tmp_dir, env) do |pid, stdout|
107
138
  begin
108
139
  waiting(timeout) do
109
- while process_exist?(pid) && !matched
140
+ while process_exist?(pid)
110
141
  readables, _, _ = IO.select([stdout], nil, nil, 1)
111
142
  next unless readables
112
143
  break if readables.first.eof?
@@ -118,6 +149,18 @@ class TestFluentdCommand < ::Test::Unit::TestCase
118
149
  if pattern_list.all?{|ptn| lines.any?{|line| ptn.is_a?(Regexp) ? ptn.match(line) : line.include?(ptn) } }
119
150
  matched = true
120
151
  end
152
+
153
+ if Fluent.windows?
154
+ # https://github.com/fluent/fluentd/issues/4095
155
+ # On Windows, the initial process is different from the supervisor process,
156
+ # so we need to wait until `SUPERVISOR_PID_PATTERN` appears in the logs to get the pid.
157
+ # (Worker processes will be killed by the supervisor process, so we don't need it-)
158
+ break if matched && SUPERVISOR_PID_PATTERN =~ stdio_buf
159
+ else
160
+ # On Non-Windows, the initial process is the supervisor process,
161
+ # so we don't need to wait `SUPERVISOR_PID_PATTERN`.
162
+ break if matched
163
+ end
121
164
  end
122
165
  end
123
166
  ensure
@@ -131,6 +174,10 @@ class TestFluentdCommand < ::Test::Unit::TestCase
131
174
  end
132
175
  rescue Timeout::Error
133
176
  assert_error_msg = "execution timeout"
177
+ # https://github.com/fluent/fluentd/issues/4095
178
+ # On Windows, timeout without `@supervisor_pid` means that the test is invalid,
179
+ # since the supervisor process will survive without being killed correctly.
180
+ flunk("Invalid test: The pid of supervisor could not be taken, which is necessary on Windows.") if Fluent.windows? && @supervisor_pid.nil?
134
181
  rescue => e
135
182
  assert_error_msg = "unexpected error in launching fluentd: #{e.inspect}"
136
183
  else
@@ -162,7 +209,7 @@ class TestFluentdCommand < ::Test::Unit::TestCase
162
209
  assert matched && !matched_wrongly, assert_error_msg
163
210
  end
164
211
 
165
- def assert_fluentd_fails_to_start(cmdline, *pattern_list, timeout: 10)
212
+ def assert_fluentd_fails_to_start(cmdline, *pattern_list, timeout: 20)
166
213
  # empty_list.all?{ ... } is always true
167
214
  matched = false
168
215
  running = false
@@ -198,6 +245,10 @@ class TestFluentdCommand < ::Test::Unit::TestCase
198
245
  end
199
246
  rescue Timeout::Error
200
247
  assert_error_msg = "execution timeout with command out:\n" + stdio_buf
248
+ # https://github.com/fluent/fluentd/issues/4095
249
+ # On Windows, timeout without `@supervisor_pid` means that the test is invalid,
250
+ # since the supervisor process will survive without being killed correctly.
251
+ flunk("Invalid test: The pid of supervisor could not be taken, which is necessary on Windows.") if Fluent.windows? && @supervisor_pid.nil?
201
252
  rescue => e
202
253
  assert_error_msg = "unexpected error in launching fluentd: #{e.inspect}\n" + stdio_buf
203
254
  assert false, assert_error_msg
@@ -269,7 +320,7 @@ CONF
269
320
 
270
321
  sub_test_case 'with system configuration about root directory' do
271
322
  setup do
272
- @root_path = File.join(TMP_DIR, "rootpath")
323
+ @root_path = File.join(@tmp_dir, "rootpath")
273
324
  FileUtils.rm_rf(@root_path)
274
325
  @conf = <<CONF
275
326
  <system>
@@ -308,7 +359,7 @@ CONF
308
359
  end
309
360
 
310
361
  test 'fails to launch fluentd if specified root path is invalid path for directory' do
311
- File.open(@root_path, 'w') do |_|
362
+ Fluent::FileWrapper.open(@root_path, 'w') do |_|
312
363
  # create file and close it
313
364
  end
314
365
  conf_path = create_conf_file('existing_root_dir.conf', @conf)
@@ -508,7 +559,7 @@ CONF
508
559
 
509
560
  assert_fluentd_fails_to_start(
510
561
  create_cmdline(conf_path, "-p", File.dirname(plugin_path)),
511
- "in_buggy.rb:5: syntax error, unexpected end-of-input, expecting"
562
+ "in_buggy.rb:5: syntax error, unexpected end-of-input"
512
563
  )
513
564
  end
514
565
  end
@@ -554,7 +605,7 @@ CONF
554
605
 
555
606
  sub_test_case 'configured to run 2 workers' do
556
607
  setup do
557
- @root_path = File.join(TMP_DIR, "rootpath")
608
+ @root_path = File.join(@tmp_dir, "rootpath")
558
609
  FileUtils.rm_rf(@root_path)
559
610
  FileUtils.mkdir_p(@root_path)
560
611
  end
@@ -589,6 +640,39 @@ CONF
589
640
  )
590
641
  end
591
642
 
643
+ sub_test_case "YAML config format" do
644
+ test 'success to start the number of workers specified in configuration' do
645
+ conf = <<'CONF'
646
+ system:
647
+ workers: 2
648
+ root_dir: "#{@root_path}"
649
+ config:
650
+ - source:
651
+ $type: dummy
652
+ $id: !fluent/s "dummy.#{worker_id}" # check worker_id works or not with actual command
653
+ $label: '@dummydata'
654
+ tag: dummy
655
+ dummy: !fluent/json {"message": !fluent/s "yay from #{hostname}!"}
656
+
657
+ - label:
658
+ $name: '@dummydata'
659
+ config:
660
+ - match:
661
+ $tag: dummy
662
+ $type: "null"
663
+ $id: blackhole
664
+ CONF
665
+ conf_path = create_conf_file('workers1.yaml', conf)
666
+ assert Dir.exist?(@root_path)
667
+
668
+ assert_log_matches(
669
+ create_cmdline(conf_path),
670
+ "#0 fluentd worker is now running worker=0",
671
+ "#1 fluentd worker is now running worker=1"
672
+ )
673
+ end
674
+ end
675
+
592
676
  test 'success to start the number of workers specified by command line option' do
593
677
  conf = <<CONF
594
678
  <system>
@@ -857,7 +941,7 @@ CONF
857
941
  '-external-encoding' => '--external-encoding=utf-8',
858
942
  '-internal-encoding' => '--internal-encoding=utf-8',
859
943
  )
860
- test "-E option is set to RUBYOPT" do |opt|
944
+ test "-E option is set to RUBYOPT" do |base_opt|
861
945
  conf = <<CONF
862
946
  <source>
863
947
  @type dummy
@@ -868,6 +952,7 @@ CONF
868
952
  </match>
869
953
  CONF
870
954
  conf_path = create_conf_file('rubyopt_test.conf', conf)
955
+ opt = base_opt.dup
871
956
  opt << " #{ENV['RUBYOPT']}" if ENV['RUBYOPT']
872
957
  assert_log_matches(
873
958
  create_cmdline(conf_path),
@@ -907,9 +992,14 @@ CONF
907
992
  </match>
908
993
  CONF
909
994
  conf_path = create_conf_file('rubyopt_invalid_test.conf', conf)
995
+ if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('3.3.0')
996
+ expected_phrase = 'ruby: invalid switch in RUBYOPT'
997
+ else
998
+ expected_phrase = 'Invalid option is passed to RUBYOPT'
999
+ end
910
1000
  assert_log_matches(
911
1001
  create_cmdline(conf_path),
912
- 'Invalid option is passed to RUBYOPT',
1002
+ expected_phrase,
913
1003
  env: { 'RUBYOPT' => 'a' },
914
1004
  )
915
1005
  end
@@ -928,10 +1018,10 @@ CONF
928
1018
  </match>
929
1019
  CONF
930
1020
  ruby_path = ServerEngine.ruby_bin_path
931
- tmp_ruby_path = File.join(TMP_DIR, "ruby with spaces")
1021
+ tmp_ruby_path = File.join(@tmp_dir, "ruby with spaces")
932
1022
  if Fluent.windows?
933
1023
  tmp_ruby_path << ".bat"
934
- File.open(tmp_ruby_path, "w") do |file|
1024
+ Fluent::FileWrapper.open(tmp_ruby_path, "w") do |file|
935
1025
  file.write "#{ruby_path} %*"
936
1026
  end
937
1027
  else
@@ -1103,4 +1193,99 @@ CONF
1103
1193
  "shared socket for multiple workers is disabled",)
1104
1194
  end
1105
1195
  end
1196
+
1197
+ # TODO: `patterns_not_match` can test only logs up to `pattern_list`,
1198
+ # so we need to fix some meaningless `patterns_not_match` conditions.
1199
+ sub_test_case 'log_level by command line option' do
1200
+ test 'info' do
1201
+ conf = ""
1202
+ conf_path = create_conf_file('empty.conf', conf)
1203
+ assert File.exist?(conf_path)
1204
+ assert_log_matches(create_cmdline(conf_path),
1205
+ "[info]",
1206
+ patterns_not_match: ["[debug]"])
1207
+ end
1208
+
1209
+ test 'debug' do
1210
+ conf = ""
1211
+ conf_path = create_conf_file('empty.conf', conf)
1212
+ assert File.exist?(conf_path)
1213
+ assert_log_matches(create_cmdline(conf_path, "-v"),
1214
+ "[debug]",
1215
+ patterns_not_match: ["[trace]"])
1216
+ end
1217
+
1218
+ data("Trace" => "-vv")
1219
+ data("Invalid low level should be treated as Trace level": "-vvv")
1220
+ test 'trace' do |option|
1221
+ conf = <<CONF
1222
+ <source>
1223
+ @type sample
1224
+ tag test
1225
+ </source>
1226
+ CONF
1227
+ conf_path = create_conf_file('sample.conf', conf)
1228
+ assert File.exist?(conf_path)
1229
+ assert_log_matches(create_cmdline(conf_path, option),
1230
+ "[trace]",)
1231
+ end
1232
+
1233
+ test 'warn' do
1234
+ omit "Can't run on Windows since there is no way to take pid of the supervisor." if Fluent.windows?
1235
+ conf = <<CONF
1236
+ <source>
1237
+ @type sample
1238
+ tag test
1239
+ </source>
1240
+ CONF
1241
+ conf_path = create_conf_file('sample.conf', conf)
1242
+ assert File.exist?(conf_path)
1243
+ assert_log_matches(create_cmdline(conf_path, "-q"),
1244
+ "[warn]",
1245
+ patterns_not_match: ["[info]"])
1246
+ end
1247
+
1248
+ data("Error" => "-qq")
1249
+ data("Fatal should be treated as Error level" => "-qqq")
1250
+ data("Invalid high level should be treated as Error level": "-qqqq")
1251
+ test 'error' do |option|
1252
+ # This test can run on Windows correctly,
1253
+ # since the process will stop automatically with an error.
1254
+ conf = <<CONF
1255
+ <source>
1256
+ @type plugin_not_found
1257
+ tag test
1258
+ </source>
1259
+ CONF
1260
+ conf_path = create_conf_file('plugin_not_found.conf', conf)
1261
+ assert File.exist?(conf_path)
1262
+ assert_log_matches(create_cmdline(conf_path, option),
1263
+ "[error]",
1264
+ patterns_not_match: ["[warn]"])
1265
+ end
1266
+
1267
+ test 'system config one should not be overwritten when cmd line one is not specified' do
1268
+ conf = <<CONF
1269
+ <system>
1270
+ log_level debug
1271
+ </system>
1272
+ CONF
1273
+ conf_path = create_conf_file('debug.conf', conf)
1274
+ assert File.exist?(conf_path)
1275
+ assert_log_matches(create_cmdline(conf_path),
1276
+ "[debug]")
1277
+ end
1278
+ end
1279
+
1280
+ sub_test_case "inline_config" do
1281
+ test "can change log_level by --inline-config" do
1282
+ # Since we can't define multiple `<system>` directives, this use-case is not recommended.
1283
+ # This is just for this test.
1284
+ inline_conf = '<system>\nlog_level debug\n</system>'
1285
+ conf_path = create_conf_file('test.conf', "")
1286
+ assert File.exist?(conf_path)
1287
+ assert_log_matches(create_cmdline(conf_path, "--inline-config", inline_conf),
1288
+ "[debug]")
1289
+ end
1290
+ end
1106
1291
  end
@@ -188,7 +188,6 @@ slow_flush_log_threshold: float: (20.0)
188
188
  retry_exponential_backoff_base: float: (2)
189
189
  retry_max_interval: time: (nil)
190
190
  retry_randomize: bool: (true)
191
- disable_chunk_backup: bool: (false)
192
191
  <secondary>: optional, single
193
192
  @type: string: (nil)
194
193
  <buffer>: optional, single
@@ -22,7 +22,7 @@ class TextParserTest < ::Test::Unit::TestCase
22
22
  Fluent::TextParser.register_template('multi_event_test', Proc.new { MultiEventTestParser.new })
23
23
 
24
24
  def test_lookup_unknown_format
25
- assert_raise Fluent::ConfigError do
25
+ assert_raise Fluent::NotFoundPluginError do
26
26
  Fluent::Plugin.new_parser('unknown')
27
27
  end
28
28
  end
@@ -38,14 +38,14 @@ class TextParserTest < ::Test::Unit::TestCase
38
38
 
39
39
  def test_parse_with_return
40
40
  parser = Fluent::TextParser.new
41
- parser.configure('format' => 'none')
41
+ parser.configure(config_element('test', '', 'format' => 'none'))
42
42
  _time, record = parser.parse('log message!')
43
43
  assert_equal({'message' => 'log message!'}, record)
44
44
  end
45
45
 
46
46
  def test_parse_with_block
47
47
  parser = Fluent::TextParser.new
48
- parser.configure('format' => 'none')
48
+ parser.configure(config_element('test', '', 'format' => 'none'))
49
49
  parser.parse('log message!') { |time, record|
50
50
  assert_equal({'message' => 'log message!'}, record)
51
51
  }
@@ -53,7 +53,7 @@ class TextParserTest < ::Test::Unit::TestCase
53
53
 
54
54
  def test_multi_event_parser
55
55
  parser = Fluent::TextParser.new
56
- parser.configure('format' => 'multi_event_test')
56
+ parser.configure(config_element('test', '', 'format' => 'multi_event_test'))
57
57
  i = 0
58
58
  parser.parse('log message!') { |time, record|
59
59
  assert_equal('log message!', record['message'])
@@ -67,7 +67,7 @@ class TextParserTest < ::Test::Unit::TestCase
67
67
  assert_nil p1.estimate_current_event
68
68
  assert_nil p1.parser
69
69
 
70
- p1.configure('format' => 'none')
70
+ p1.configure(config_element('test', '', 'format' => 'none'))
71
71
  assert_equal true, p1.parser.estimate_current_event
72
72
 
73
73
  p2 = Fluent::TextParser.new
@@ -76,7 +76,7 @@ class TextParserTest < ::Test::Unit::TestCase
76
76
 
77
77
  p2.estimate_current_event = false
78
78
 
79
- p2.configure('format' => 'none')
79
+ p2.configure(config_element('test', '', 'format' => 'none'))
80
80
  assert_equal false, p2.parser.estimate_current_event
81
81
  end
82
82
 
@@ -5,13 +5,6 @@ require 'fluent/config/section'
5
5
  require 'fluent/system_config'
6
6
 
7
7
  module Fluent::Config
8
- class FakeLoggerInitializer
9
- attr_accessor :level
10
- def initialize
11
- @level = nil
12
- end
13
- end
14
-
15
8
  class FakeSupervisor
16
9
  attr_writer :log_level
17
10
 
@@ -19,8 +12,8 @@ module Fluent::Config
19
12
  @system_config = nil
20
13
  @cl_opt = {
21
14
  wokers: nil,
15
+ restart_worker_interval: nil,
22
16
  root_dir: nil,
23
- log: FakeLoggerInitializer.new,
24
17
  log_level: Fluent::Log::LEVEL_INFO,
25
18
  suppress_interval: nil,
26
19
  suppress_config_dump: nil,
@@ -28,6 +21,8 @@ module Fluent::Config
28
21
  log_event_label: nil,
29
22
  log_event_verbose: nil,
30
23
  without_source: nil,
24
+ enable_input_metrics: nil,
25
+ enable_size_metrics: nil,
31
26
  emit_error_log_interval: nil,
32
27
  file_permission: nil,
33
28
  dir_permission: nil,
@@ -70,6 +65,7 @@ module Fluent::Config
70
65
  sc = Fluent::SystemConfig.new(conf)
71
66
  sc.overwrite_variables(**s.for_system_config)
72
67
  assert_equal(1, sc.workers)
68
+ assert_equal(0, sc.restart_worker_interval)
73
69
  assert_nil(sc.root_dir)
74
70
  assert_equal(Fluent::Log::LEVEL_INFO, sc.log_level)
75
71
  assert_nil(sc.suppress_repeated_stacktrace)
@@ -77,13 +73,17 @@ module Fluent::Config
77
73
  assert_nil(sc.emit_error_log_interval)
78
74
  assert_nil(sc.suppress_config_dump)
79
75
  assert_nil(sc.without_source)
76
+ assert_nil(sc.enable_input_metrics)
77
+ assert_nil(sc.enable_size_metrics)
80
78
  assert_nil(sc.enable_msgpack_time_support)
79
+ assert(!sc.enable_jit)
81
80
  assert_equal(:text, sc.log.format)
82
81
  assert_equal('%Y-%m-%d %H:%M:%S %z', sc.log.time_format)
83
82
  end
84
83
 
85
84
  data(
86
85
  'workers' => ['workers', 3],
86
+ 'restart_worker_interval' => ['restart_worker_interval', 60],
87
87
  'root_dir' => ['root_dir', File.join(TMP_DIR, 'root')],
88
88
  'log_level' => ['log_level', 'error'],
89
89
  'suppress_repeated_stacktrace' => ['suppress_repeated_stacktrace', true],
@@ -93,6 +93,9 @@ module Fluent::Config
93
93
  'without_source' => ['without_source', true],
94
94
  'strict_config_value' => ['strict_config_value', true],
95
95
  'enable_msgpack_time_support' => ['enable_msgpack_time_support', true],
96
+ 'enable_input_metrics' => ['enable_input_metrics', true],
97
+ 'enable_size_metrics' => ['enable_size_metrics', true],
98
+ 'enable_jit' => ['enable_jit', true],
96
99
  )
97
100
  test "accepts parameters" do |(k, v)|
98
101
  conf = parse_text(<<-EOS)
@@ -148,7 +151,7 @@ module Fluent::Config
148
151
  data('daily' => "daily",
149
152
  'weekly' => 'weekly',
150
153
  'monthly' => 'monthly')
151
- test "symbols for rotate_age" do |age|
154
+ test "strings for rotate_age" do |age|
152
155
  conf = parse_text(<<-EOS)
153
156
  <system>
154
157
  <log>
@@ -157,7 +160,7 @@ module Fluent::Config
157
160
  </system>
158
161
  EOS
159
162
  sc = Fluent::SystemConfig.new(conf)
160
- assert_equal(age.to_sym, sc.log.rotate_age)
163
+ assert_equal(age, sc.log.rotate_age)
161
164
  end
162
165
 
163
166
  test "numeric number for rotate age" do
@@ -168,7 +171,6 @@ module Fluent::Config
168
171
  </log>
169
172
  </system>
170
173
  EOS
171
- s = FakeSupervisor.new
172
174
  sc = Fluent::SystemConfig.new(conf)
173
175
  assert_equal(3, sc.log.rotate_age)
174
176
  end
@@ -190,7 +190,7 @@ class TestConfigTypes < ::Test::Unit::TestCase
190
190
  data("val" => [:val, 'val'],
191
191
  "v" => [:v, 'v'],
192
192
  "value" => [:value, 'value'])
193
- test 'enum' do |(expected, val, list)|
193
+ test 'enum' do |(expected, val)|
194
194
  assert_equal expected, Config::ENUM_TYPE.call(val, {list: [:val, :value, :v]})
195
195
  end
196
196
 
@@ -0,0 +1,110 @@
1
+ require_relative '../helper'
2
+
3
+ require 'fluent/log'
4
+ require 'fluent/log/console_adapter'
5
+
6
+ class ConsoleAdapterTest < Test::Unit::TestCase
7
+ def setup
8
+ @timestamp = Time.parse("2023-01-01 15:32:41 +0000")
9
+ @timestamp_str = @timestamp.strftime("%Y-%m-%d %H:%M:%S %z")
10
+ Timecop.freeze(@timestamp)
11
+
12
+ @logdev = Fluent::Test::DummyLogDevice.new
13
+ @logger = ServerEngine::DaemonLogger.new(@logdev)
14
+ @fluent_log = Fluent::Log.new(@logger)
15
+ @console_logger = Fluent::Log::ConsoleAdapter.wrap(@fluent_log)
16
+ end
17
+
18
+ def teardown
19
+ Timecop.return
20
+ end
21
+
22
+ def test_expected_log_levels
23
+ assert_equal({debug: 0, info: 1, warn: 2, error: 3, fatal: 4},
24
+ Console::Logger::LEVELS)
25
+ end
26
+
27
+ data(trace: [Fluent::Log::LEVEL_TRACE, :debug],
28
+ debug: [Fluent::Log::LEVEL_DEBUG, :debug],
29
+ info: [Fluent::Log::LEVEL_INFO, :info],
30
+ warn: [Fluent::Log::LEVEL_WARN, :warn],
31
+ error: [Fluent::Log::LEVEL_ERROR, :error],
32
+ fatal: [Fluent::Log::LEVEL_FATAL, :fatal])
33
+ def test_reflect_log_level(data)
34
+ level, expected = data
35
+ @fluent_log.level = level
36
+ console_logger = Fluent::Log::ConsoleAdapter.wrap(@fluent_log)
37
+ assert_equal(Console::Logger::LEVELS[expected],
38
+ console_logger.level)
39
+ end
40
+
41
+ data(debug: :debug,
42
+ info: :info,
43
+ warn: :warn,
44
+ error: :error,
45
+ fatal: :fatal)
46
+ def test_string_subject(level)
47
+ @console_logger.send(level, "subject")
48
+ assert_equal(["#{@timestamp_str} [#{level}]: 0.0s: subject\n"],
49
+ @logdev.logs)
50
+ end
51
+
52
+ data(debug: :debug,
53
+ info: :info,
54
+ warn: :warn,
55
+ error: :error,
56
+ fatal: :fatal)
57
+ def test_args(level)
58
+ @console_logger.send(level, "subject", 1, 2, 3)
59
+ assert_equal([
60
+ "#{@timestamp_str} [#{level}]: 0.0s: subject\n" +
61
+ " | 1\n" +
62
+ " | 2\n" +
63
+ " | 3\n"
64
+ ],
65
+ @logdev.logs)
66
+ end
67
+
68
+ data(debug: :debug,
69
+ info: :info,
70
+ warn: :warn,
71
+ error: :error,
72
+ fatal: :fatal)
73
+ def test_options(level)
74
+ @console_logger.send(level, "subject", kwarg1: "opt1", kwarg2: "opt2")
75
+ assert_equal([
76
+ "#{@timestamp_str} [#{level}]: 0.0s: subject\n" +
77
+ " | {\"kwarg1\":\"opt1\",\"kwarg2\":\"opt2\"}\n"
78
+ ],
79
+ @logdev.logs)
80
+ end
81
+
82
+ data(debug: :debug,
83
+ info: :info,
84
+ warn: :warn,
85
+ error: :error,
86
+ fatal: :fatal)
87
+ def test_block(level)
88
+ @console_logger.send(level, "subject") { "block message" }
89
+ assert_equal([
90
+ "#{@timestamp_str} [#{level}]: 0.0s: subject\n" +
91
+ " | block message\n"
92
+ ],
93
+ @logdev.logs)
94
+ end
95
+
96
+ data(debug: :debug,
97
+ info: :info,
98
+ warn: :warn,
99
+ error: :error,
100
+ fatal: :fatal)
101
+ def test_multiple_entries(level)
102
+ @console_logger.send(level, "subject1")
103
+ @console_logger.send(level, "line2")
104
+ assert_equal([
105
+ "#{@timestamp_str} [#{level}]: 0.0s: subject1\n",
106
+ "#{@timestamp_str} [#{level}]: 0.0s: line2\n"
107
+ ],
108
+ @logdev.logs)
109
+ end
110
+ end