fluentd 1.13.3 → 1.16.5

Sign up to get free protection for your applications and to get access to all the features.
Files changed (179) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/{bug_report.yaml → bug_report.yml} +2 -0
  3. data/.github/ISSUE_TEMPLATE/config.yml +2 -2
  4. data/.github/ISSUE_TEMPLATE/{feature_request.yaml → feature_request.yml} +1 -0
  5. data/.github/workflows/stale-actions.yml +11 -9
  6. data/.github/workflows/test.yml +32 -0
  7. data/CHANGELOG.md +490 -10
  8. data/CONTRIBUTING.md +2 -2
  9. data/MAINTAINERS.md +7 -5
  10. data/README.md +3 -23
  11. data/Rakefile +1 -1
  12. data/SECURITY.md +14 -0
  13. data/fluentd.gemspec +7 -8
  14. data/lib/fluent/command/cat.rb +13 -3
  15. data/lib/fluent/command/ctl.rb +6 -3
  16. data/lib/fluent/command/fluentd.rb +73 -65
  17. data/lib/fluent/command/plugin_config_formatter.rb +1 -1
  18. data/lib/fluent/compat/output.rb +9 -6
  19. data/lib/fluent/config/dsl.rb +1 -1
  20. data/lib/fluent/config/error.rb +12 -0
  21. data/lib/fluent/config/literal_parser.rb +2 -2
  22. data/lib/fluent/config/parser.rb +1 -1
  23. data/lib/fluent/config/v1_parser.rb +3 -3
  24. data/lib/fluent/config/yaml_parser/fluent_value.rb +47 -0
  25. data/lib/fluent/config/yaml_parser/loader.rb +108 -0
  26. data/lib/fluent/config/yaml_parser/parser.rb +166 -0
  27. data/lib/fluent/config/yaml_parser/section_builder.rb +107 -0
  28. data/lib/fluent/config/yaml_parser.rb +56 -0
  29. data/lib/fluent/config.rb +14 -1
  30. data/lib/fluent/counter/server.rb +1 -1
  31. data/lib/fluent/counter/validator.rb +3 -3
  32. data/lib/fluent/daemon.rb +2 -4
  33. data/lib/fluent/engine.rb +1 -1
  34. data/lib/fluent/env.rb +4 -0
  35. data/lib/fluent/error.rb +3 -0
  36. data/lib/fluent/event.rb +8 -4
  37. data/lib/fluent/event_router.rb +47 -2
  38. data/lib/fluent/file_wrapper.rb +137 -0
  39. data/lib/fluent/log/console_adapter.rb +66 -0
  40. data/lib/fluent/log.rb +44 -5
  41. data/lib/fluent/match.rb +1 -1
  42. data/lib/fluent/msgpack_factory.rb +6 -1
  43. data/lib/fluent/oj_options.rb +1 -2
  44. data/lib/fluent/plugin/bare_output.rb +49 -8
  45. data/lib/fluent/plugin/base.rb +26 -9
  46. data/lib/fluent/plugin/buf_file.rb +34 -5
  47. data/lib/fluent/plugin/buf_file_single.rb +32 -3
  48. data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
  49. data/lib/fluent/plugin/buffer.rb +216 -70
  50. data/lib/fluent/plugin/filter.rb +35 -1
  51. data/lib/fluent/plugin/filter_record_transformer.rb +1 -1
  52. data/lib/fluent/plugin/in_forward.rb +2 -2
  53. data/lib/fluent/plugin/in_http.rb +39 -10
  54. data/lib/fluent/plugin/in_monitor_agent.rb +4 -2
  55. data/lib/fluent/plugin/in_sample.rb +1 -1
  56. data/lib/fluent/plugin/in_syslog.rb +13 -1
  57. data/lib/fluent/plugin/in_tail/group_watch.rb +204 -0
  58. data/lib/fluent/plugin/in_tail/position_file.rb +33 -33
  59. data/lib/fluent/plugin/in_tail.rb +216 -84
  60. data/lib/fluent/plugin/in_tcp.rb +47 -2
  61. data/lib/fluent/plugin/input.rb +39 -1
  62. data/lib/fluent/plugin/metrics.rb +119 -0
  63. data/lib/fluent/plugin/metrics_local.rb +96 -0
  64. data/lib/fluent/plugin/multi_output.rb +43 -6
  65. data/lib/fluent/plugin/out_copy.rb +1 -1
  66. data/lib/fluent/plugin/out_exec_filter.rb +2 -2
  67. data/lib/fluent/plugin/out_file.rb +20 -2
  68. data/lib/fluent/plugin/out_forward/ack_handler.rb +19 -4
  69. data/lib/fluent/plugin/out_forward/socket_cache.rb +2 -0
  70. data/lib/fluent/plugin/out_forward.rb +17 -9
  71. data/lib/fluent/plugin/out_secondary_file.rb +39 -22
  72. data/lib/fluent/plugin/output.rb +167 -78
  73. data/lib/fluent/plugin/parser.rb +3 -4
  74. data/lib/fluent/plugin/parser_apache2.rb +1 -1
  75. data/lib/fluent/plugin/parser_json.rb +1 -1
  76. data/lib/fluent/plugin/parser_syslog.rb +1 -1
  77. data/lib/fluent/plugin/storage_local.rb +3 -5
  78. data/lib/fluent/plugin.rb +10 -1
  79. data/lib/fluent/plugin_helper/child_process.rb +3 -0
  80. data/lib/fluent/plugin_helper/event_emitter.rb +8 -1
  81. data/lib/fluent/plugin_helper/event_loop.rb +2 -2
  82. data/lib/fluent/plugin_helper/http_server/server.rb +2 -1
  83. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  84. data/lib/fluent/plugin_helper/record_accessor.rb +1 -1
  85. data/lib/fluent/plugin_helper/retry_state.rb +14 -4
  86. data/lib/fluent/plugin_helper/server.rb +35 -6
  87. data/lib/fluent/plugin_helper/service_discovery.rb +2 -2
  88. data/lib/fluent/plugin_helper/socket.rb +13 -2
  89. data/lib/fluent/plugin_helper/thread.rb +3 -3
  90. data/lib/fluent/plugin_helper.rb +1 -0
  91. data/lib/fluent/plugin_id.rb +3 -2
  92. data/lib/fluent/registry.rb +2 -1
  93. data/lib/fluent/root_agent.rb +6 -0
  94. data/lib/fluent/rpc.rb +4 -3
  95. data/lib/fluent/supervisor.rb +283 -259
  96. data/lib/fluent/system_config.rb +13 -3
  97. data/lib/fluent/test/driver/base.rb +11 -5
  98. data/lib/fluent/test/driver/filter.rb +4 -0
  99. data/lib/fluent/test/startup_shutdown.rb +6 -8
  100. data/lib/fluent/time.rb +21 -20
  101. data/lib/fluent/version.rb +1 -1
  102. data/lib/fluent/win32api.rb +38 -0
  103. data/lib/fluent/winsvc.rb +5 -8
  104. data/templates/new_gem/test/helper.rb.erb +0 -1
  105. data/test/command/test_cat.rb +31 -2
  106. data/test/command/test_ctl.rb +1 -2
  107. data/test/command/test_fluentd.rb +209 -24
  108. data/test/command/test_plugin_config_formatter.rb +0 -1
  109. data/test/compat/test_parser.rb +6 -6
  110. data/test/config/test_system_config.rb +13 -11
  111. data/test/config/test_types.rb +1 -1
  112. data/test/log/test_console_adapter.rb +110 -0
  113. data/test/plugin/in_tail/test_io_handler.rb +26 -8
  114. data/test/plugin/in_tail/test_position_file.rb +48 -59
  115. data/test/plugin/out_forward/test_ack_handler.rb +39 -0
  116. data/test/plugin/out_forward/test_socket_cache.rb +26 -1
  117. data/test/plugin/test_bare_output.rb +14 -1
  118. data/test/plugin/test_base.rb +133 -1
  119. data/test/plugin/test_buf_file.rb +62 -23
  120. data/test/plugin/test_buf_file_single.rb +65 -0
  121. data/test/plugin/test_buffer.rb +267 -3
  122. data/test/plugin/test_buffer_chunk.rb +11 -0
  123. data/test/plugin/test_filter.rb +12 -1
  124. data/test/plugin/test_filter_parser.rb +1 -1
  125. data/test/plugin/test_filter_stdout.rb +2 -2
  126. data/test/plugin/test_in_forward.rb +9 -11
  127. data/test/plugin/test_in_http.rb +65 -3
  128. data/test/plugin/test_in_monitor_agent.rb +216 -11
  129. data/test/plugin/test_in_object_space.rb +9 -3
  130. data/test/plugin/test_in_syslog.rb +35 -0
  131. data/test/plugin/test_in_tail.rb +1393 -385
  132. data/test/plugin/test_in_tcp.rb +87 -2
  133. data/test/plugin/test_in_udp.rb +28 -0
  134. data/test/plugin/test_in_unix.rb +2 -2
  135. data/test/plugin/test_input.rb +12 -1
  136. data/test/plugin/test_metrics.rb +294 -0
  137. data/test/plugin/test_metrics_local.rb +96 -0
  138. data/test/plugin/test_multi_output.rb +25 -1
  139. data/test/plugin/test_out_exec.rb +6 -4
  140. data/test/plugin/test_out_exec_filter.rb +6 -2
  141. data/test/plugin/test_out_file.rb +34 -17
  142. data/test/plugin/test_out_forward.rb +78 -77
  143. data/test/plugin/test_out_http.rb +1 -0
  144. data/test/plugin/test_out_stdout.rb +2 -2
  145. data/test/plugin/test_output.rb +297 -12
  146. data/test/plugin/test_output_as_buffered.rb +44 -44
  147. data/test/plugin/test_output_as_buffered_compress.rb +32 -18
  148. data/test/plugin/test_output_as_buffered_retries.rb +54 -7
  149. data/test/plugin/test_output_as_buffered_secondary.rb +4 -4
  150. data/test/plugin/test_parser_regexp.rb +1 -6
  151. data/test/plugin/test_parser_syslog.rb +1 -1
  152. data/test/plugin_helper/test_cert_option.rb +1 -1
  153. data/test/plugin_helper/test_child_process.rb +38 -16
  154. data/test/plugin_helper/test_event_emitter.rb +29 -0
  155. data/test/plugin_helper/test_http_server_helper.rb +1 -1
  156. data/test/plugin_helper/test_metrics.rb +137 -0
  157. data/test/plugin_helper/test_retry_state.rb +602 -38
  158. data/test/plugin_helper/test_server.rb +78 -6
  159. data/test/plugin_helper/test_timer.rb +2 -2
  160. data/test/test_config.rb +191 -24
  161. data/test/test_event_router.rb +17 -0
  162. data/test/test_file_wrapper.rb +53 -0
  163. data/test/test_formatter.rb +24 -21
  164. data/test/test_log.rb +122 -40
  165. data/test/test_msgpack_factory.rb +32 -0
  166. data/test/test_plugin_classes.rb +102 -0
  167. data/test/test_root_agent.rb +30 -1
  168. data/test/test_supervisor.rb +477 -257
  169. data/test/test_time_parser.rb +22 -0
  170. metadata +55 -34
  171. data/.drone.yml +0 -35
  172. data/.github/workflows/issue-auto-closer.yml +0 -12
  173. data/.github/workflows/linux-test.yaml +0 -36
  174. data/.github/workflows/macos-test.yaml +0 -30
  175. data/.github/workflows/windows-test.yaml +0 -46
  176. data/.gitlab-ci.yml +0 -103
  177. data/lib/fluent/plugin/file_wrapper.rb +0 -187
  178. data/test/plugin/test_file_wrapper.rb +0 -126
  179. data/test/test_logger_initializer.rb +0 -46
@@ -5,20 +5,35 @@ require_relative '../helper'
5
5
 
6
6
  require 'fileutils'
7
7
  require 'timeout'
8
+ require 'securerandom'
9
+ require 'fluent/file_wrapper'
8
10
 
9
11
  class TestFluentdCommand < ::Test::Unit::TestCase
10
- TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/command/fluentd#{ENV['TEST_ENV_NUMBER']}")
11
12
  SUPERVISOR_PID_PATTERN = /starting fluentd-[.0-9]+ pid=(\d+)/
12
13
  WORKER_PID_PATTERN = /starting fluentd worker pid=(\d+) /
13
14
 
15
+ def tmp_dir
16
+ File.join(File.dirname(__FILE__), "..", "tmp", "command" "fluentd#{ENV['TEST_ENV_NUMBER']}", SecureRandom.hex(10))
17
+ end
18
+
14
19
  setup do
15
- FileUtils.rm_rf(TMP_DIR)
16
- FileUtils.mkdir_p(TMP_DIR)
20
+ @tmp_dir = tmp_dir
21
+ FileUtils.mkdir_p(@tmp_dir)
17
22
  @supervisor_pid = nil
18
23
  @worker_pids = []
19
24
  ENV["TEST_RUBY_PATH"] = nil
20
25
  end
21
26
 
27
+ teardown do
28
+ begin
29
+ FileUtils.rm_rf(@tmp_dir)
30
+ rescue Errno::EACCES
31
+ # It may occur on Windows because of delete pending state due to delayed GC.
32
+ # Ruby 3.2 or later doesn't ignore Errno::EACCES:
33
+ # https://github.com/ruby/ruby/commit/983115cf3c8f75b1afbe3274f02c1529e1ce3a81
34
+ end
35
+ end
36
+
22
37
  def process_exist?(pid)
23
38
  begin
24
39
  r = Process.waitpid(pid, Process::WNOHANG)
@@ -30,17 +45,17 @@ class TestFluentdCommand < ::Test::Unit::TestCase
30
45
  end
31
46
 
32
47
  def create_conf_file(name, content, ext_enc = 'utf-8')
33
- conf_path = File.join(TMP_DIR, name)
34
- File.open(conf_path, "w:#{ext_enc}:utf-8") do |file|
48
+ conf_path = File.join(@tmp_dir, name)
49
+ Fluent::FileWrapper.open(conf_path, "w:#{ext_enc}:utf-8") do |file|
35
50
  file.write content
36
51
  end
37
52
  conf_path
38
53
  end
39
54
 
40
55
  def create_plugin_file(name, content)
41
- file_path = File.join(TMP_DIR, 'plugin', name)
56
+ file_path = File.join(@tmp_dir, 'plugin', name)
42
57
  FileUtils.mkdir_p(File.dirname(file_path))
43
- File.open(file_path, 'w') do |file|
58
+ Fluent::FileWrapper.open(file_path, 'w') do |file|
44
59
  file.write content
45
60
  end
46
61
  file_path
@@ -56,8 +71,22 @@ class TestFluentdCommand < ::Test::Unit::TestCase
56
71
  end
57
72
  end
58
73
 
59
- def execute_command(cmdline, chdir=TMP_DIR, env = {})
60
- null_stream = File.open(File::NULL, 'w')
74
+ def process_kill(pid)
75
+ if Fluent.windows?
76
+ Process.kill(:KILL, pid) rescue nil
77
+ return
78
+ end
79
+
80
+ begin
81
+ Process.kill(:TERM, pid) rescue nil
82
+ Timeout.timeout(10){ sleep 0.1 while process_exist?(pid) }
83
+ rescue Timeout::Error
84
+ Process.kill(:KILL, pid) rescue nil
85
+ end
86
+ end
87
+
88
+ def execute_command(cmdline, chdir=@tmp_dir, env = {})
89
+ null_stream = Fluent::FileWrapper.open(File::NULL, 'w')
61
90
  gemfile_path = File.expand_path(File.dirname(__FILE__) + "../../../Gemfile")
62
91
 
63
92
  env = { "BUNDLE_GEMFILE" => gemfile_path }.merge(env)
@@ -70,12 +99,12 @@ class TestFluentdCommand < ::Test::Unit::TestCase
70
99
  yield pid, io
71
100
  # p(here: "execute command", pid: pid, worker_pids: @worker_pids)
72
101
  ensure
73
- Process.kill(:KILL, pid) rescue nil
102
+ process_kill(pid)
74
103
  if @supervisor_pid
75
- Process.kill(:KILL, @supervisor_pid) rescue nil
104
+ process_kill(@supervisor_pid)
76
105
  end
77
106
  @worker_pids.each do |cpid|
78
- Process.kill(:KILL, cpid) rescue nil
107
+ process_kill(cpid)
79
108
  end
80
109
  # p(here: "execute command", pid: pid, exist: process_exist?(pid), worker_pids: @worker_pids, exists: @worker_pids.map{|i| process_exist?(i) })
81
110
  Timeout.timeout(10){ sleep 0.1 while process_exist?(pid) }
@@ -97,16 +126,18 @@ class TestFluentdCommand < ::Test::Unit::TestCase
97
126
  end
98
127
  end
99
128
 
100
- def assert_log_matches(cmdline, *pattern_list, patterns_not_match: [], timeout: 10, env: {})
129
+ # ATTENTION: This stops taking logs when all `pattern_list` match or timeout,
130
+ # so `patterns_not_match` can test only logs up to that point.
131
+ def assert_log_matches(cmdline, *pattern_list, patterns_not_match: [], timeout: 20, env: {})
101
132
  matched = false
102
133
  matched_wrongly = false
103
134
  assert_error_msg = ""
104
135
  stdio_buf = ""
105
136
  begin
106
- execute_command(cmdline, TMP_DIR, env) do |pid, stdout|
137
+ execute_command(cmdline, @tmp_dir, env) do |pid, stdout|
107
138
  begin
108
139
  waiting(timeout) do
109
- while process_exist?(pid) && !matched
140
+ while process_exist?(pid)
110
141
  readables, _, _ = IO.select([stdout], nil, nil, 1)
111
142
  next unless readables
112
143
  break if readables.first.eof?
@@ -118,6 +149,18 @@ class TestFluentdCommand < ::Test::Unit::TestCase
118
149
  if pattern_list.all?{|ptn| lines.any?{|line| ptn.is_a?(Regexp) ? ptn.match(line) : line.include?(ptn) } }
119
150
  matched = true
120
151
  end
152
+
153
+ if Fluent.windows?
154
+ # https://github.com/fluent/fluentd/issues/4095
155
+ # On Windows, the initial process is different from the supervisor process,
156
+ # so we need to wait until `SUPERVISOR_PID_PATTERN` appears in the logs to get the pid.
157
+ # (Worker processes will be killed by the supervisor process, so we don't need it-)
158
+ break if matched && SUPERVISOR_PID_PATTERN =~ stdio_buf
159
+ else
160
+ # On Non-Windows, the initial process is the supervisor process,
161
+ # so we don't need to wait `SUPERVISOR_PID_PATTERN`.
162
+ break if matched
163
+ end
121
164
  end
122
165
  end
123
166
  ensure
@@ -131,6 +174,10 @@ class TestFluentdCommand < ::Test::Unit::TestCase
131
174
  end
132
175
  rescue Timeout::Error
133
176
  assert_error_msg = "execution timeout"
177
+ # https://github.com/fluent/fluentd/issues/4095
178
+ # On Windows, timeout without `@supervisor_pid` means that the test is invalid,
179
+ # since the supervisor process will survive without being killed correctly.
180
+ flunk("Invalid test: The pid of supervisor could not be taken, which is necessary on Windows.") if Fluent.windows? && @supervisor_pid.nil?
134
181
  rescue => e
135
182
  assert_error_msg = "unexpected error in launching fluentd: #{e.inspect}"
136
183
  else
@@ -162,7 +209,7 @@ class TestFluentdCommand < ::Test::Unit::TestCase
162
209
  assert matched && !matched_wrongly, assert_error_msg
163
210
  end
164
211
 
165
- def assert_fluentd_fails_to_start(cmdline, *pattern_list, timeout: 10)
212
+ def assert_fluentd_fails_to_start(cmdline, *pattern_list, timeout: 20)
166
213
  # empty_list.all?{ ... } is always true
167
214
  matched = false
168
215
  running = false
@@ -198,6 +245,10 @@ class TestFluentdCommand < ::Test::Unit::TestCase
198
245
  end
199
246
  rescue Timeout::Error
200
247
  assert_error_msg = "execution timeout with command out:\n" + stdio_buf
248
+ # https://github.com/fluent/fluentd/issues/4095
249
+ # On Windows, timeout without `@supervisor_pid` means that the test is invalid,
250
+ # since the supervisor process will survive without being killed correctly.
251
+ flunk("Invalid test: The pid of supervisor could not be taken, which is necessary on Windows.") if Fluent.windows? && @supervisor_pid.nil?
201
252
  rescue => e
202
253
  assert_error_msg = "unexpected error in launching fluentd: #{e.inspect}\n" + stdio_buf
203
254
  assert false, assert_error_msg
@@ -269,7 +320,7 @@ CONF
269
320
 
270
321
  sub_test_case 'with system configuration about root directory' do
271
322
  setup do
272
- @root_path = File.join(TMP_DIR, "rootpath")
323
+ @root_path = File.join(@tmp_dir, "rootpath")
273
324
  FileUtils.rm_rf(@root_path)
274
325
  @conf = <<CONF
275
326
  <system>
@@ -308,7 +359,7 @@ CONF
308
359
  end
309
360
 
310
361
  test 'fails to launch fluentd if specified root path is invalid path for directory' do
311
- File.open(@root_path, 'w') do |_|
362
+ Fluent::FileWrapper.open(@root_path, 'w') do |_|
312
363
  # create file and close it
313
364
  end
314
365
  conf_path = create_conf_file('existing_root_dir.conf', @conf)
@@ -508,7 +559,7 @@ CONF
508
559
 
509
560
  assert_fluentd_fails_to_start(
510
561
  create_cmdline(conf_path, "-p", File.dirname(plugin_path)),
511
- "in_buggy.rb:5: syntax error, unexpected end-of-input, expecting"
562
+ "in_buggy.rb:5: syntax error, unexpected end-of-input"
512
563
  )
513
564
  end
514
565
  end
@@ -554,7 +605,7 @@ CONF
554
605
 
555
606
  sub_test_case 'configured to run 2 workers' do
556
607
  setup do
557
- @root_path = File.join(TMP_DIR, "rootpath")
608
+ @root_path = File.join(@tmp_dir, "rootpath")
558
609
  FileUtils.rm_rf(@root_path)
559
610
  FileUtils.mkdir_p(@root_path)
560
611
  end
@@ -589,6 +640,39 @@ CONF
589
640
  )
590
641
  end
591
642
 
643
+ sub_test_case "YAML config format" do
644
+ test 'success to start the number of workers specified in configuration' do
645
+ conf = <<'CONF'
646
+ system:
647
+ workers: 2
648
+ root_dir: "#{@root_path}"
649
+ config:
650
+ - source:
651
+ $type: dummy
652
+ $id: !fluent/s "dummy.#{worker_id}" # check worker_id works or not with actual command
653
+ $label: '@dummydata'
654
+ tag: dummy
655
+ dummy: !fluent/json {"message": !fluent/s "yay from #{hostname}!"}
656
+
657
+ - label:
658
+ $name: '@dummydata'
659
+ config:
660
+ - match:
661
+ $tag: dummy
662
+ $type: "null"
663
+ $id: blackhole
664
+ CONF
665
+ conf_path = create_conf_file('workers1.yaml', conf)
666
+ assert Dir.exist?(@root_path)
667
+
668
+ assert_log_matches(
669
+ create_cmdline(conf_path),
670
+ "#0 fluentd worker is now running worker=0",
671
+ "#1 fluentd worker is now running worker=1"
672
+ )
673
+ end
674
+ end
675
+
592
676
  test 'success to start the number of workers specified by command line option' do
593
677
  conf = <<CONF
594
678
  <system>
@@ -857,7 +941,7 @@ CONF
857
941
  '-external-encoding' => '--external-encoding=utf-8',
858
942
  '-internal-encoding' => '--internal-encoding=utf-8',
859
943
  )
860
- test "-E option is set to RUBYOPT" do |opt|
944
+ test "-E option is set to RUBYOPT" do |base_opt|
861
945
  conf = <<CONF
862
946
  <source>
863
947
  @type dummy
@@ -868,6 +952,7 @@ CONF
868
952
  </match>
869
953
  CONF
870
954
  conf_path = create_conf_file('rubyopt_test.conf', conf)
955
+ opt = base_opt.dup
871
956
  opt << " #{ENV['RUBYOPT']}" if ENV['RUBYOPT']
872
957
  assert_log_matches(
873
958
  create_cmdline(conf_path),
@@ -907,9 +992,14 @@ CONF
907
992
  </match>
908
993
  CONF
909
994
  conf_path = create_conf_file('rubyopt_invalid_test.conf', conf)
995
+ if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('3.3.0')
996
+ expected_phrase = 'ruby: invalid switch in RUBYOPT'
997
+ else
998
+ expected_phrase = 'Invalid option is passed to RUBYOPT'
999
+ end
910
1000
  assert_log_matches(
911
1001
  create_cmdline(conf_path),
912
- 'Invalid option is passed to RUBYOPT',
1002
+ expected_phrase,
913
1003
  env: { 'RUBYOPT' => 'a' },
914
1004
  )
915
1005
  end
@@ -928,10 +1018,10 @@ CONF
928
1018
  </match>
929
1019
  CONF
930
1020
  ruby_path = ServerEngine.ruby_bin_path
931
- tmp_ruby_path = File.join(TMP_DIR, "ruby with spaces")
1021
+ tmp_ruby_path = File.join(@tmp_dir, "ruby with spaces")
932
1022
  if Fluent.windows?
933
1023
  tmp_ruby_path << ".bat"
934
- File.open(tmp_ruby_path, "w") do |file|
1024
+ Fluent::FileWrapper.open(tmp_ruby_path, "w") do |file|
935
1025
  file.write "#{ruby_path} %*"
936
1026
  end
937
1027
  else
@@ -1103,4 +1193,99 @@ CONF
1103
1193
  "shared socket for multiple workers is disabled",)
1104
1194
  end
1105
1195
  end
1196
+
1197
+ # TODO: `patterns_not_match` can test only logs up to `pattern_list`,
1198
+ # so we need to fix some meaningless `patterns_not_match` conditions.
1199
+ sub_test_case 'log_level by command line option' do
1200
+ test 'info' do
1201
+ conf = ""
1202
+ conf_path = create_conf_file('empty.conf', conf)
1203
+ assert File.exist?(conf_path)
1204
+ assert_log_matches(create_cmdline(conf_path),
1205
+ "[info]",
1206
+ patterns_not_match: ["[debug]"])
1207
+ end
1208
+
1209
+ test 'debug' do
1210
+ conf = ""
1211
+ conf_path = create_conf_file('empty.conf', conf)
1212
+ assert File.exist?(conf_path)
1213
+ assert_log_matches(create_cmdline(conf_path, "-v"),
1214
+ "[debug]",
1215
+ patterns_not_match: ["[trace]"])
1216
+ end
1217
+
1218
+ data("Trace" => "-vv")
1219
+ data("Invalid low level should be treated as Trace level": "-vvv")
1220
+ test 'trace' do |option|
1221
+ conf = <<CONF
1222
+ <source>
1223
+ @type sample
1224
+ tag test
1225
+ </source>
1226
+ CONF
1227
+ conf_path = create_conf_file('sample.conf', conf)
1228
+ assert File.exist?(conf_path)
1229
+ assert_log_matches(create_cmdline(conf_path, option),
1230
+ "[trace]",)
1231
+ end
1232
+
1233
+ test 'warn' do
1234
+ omit "Can't run on Windows since there is no way to take pid of the supervisor." if Fluent.windows?
1235
+ conf = <<CONF
1236
+ <source>
1237
+ @type sample
1238
+ tag test
1239
+ </source>
1240
+ CONF
1241
+ conf_path = create_conf_file('sample.conf', conf)
1242
+ assert File.exist?(conf_path)
1243
+ assert_log_matches(create_cmdline(conf_path, "-q"),
1244
+ "[warn]",
1245
+ patterns_not_match: ["[info]"])
1246
+ end
1247
+
1248
+ data("Error" => "-qq")
1249
+ data("Fatal should be treated as Error level" => "-qqq")
1250
+ data("Invalid high level should be treated as Error level": "-qqqq")
1251
+ test 'error' do |option|
1252
+ # This test can run on Windows correctly,
1253
+ # since the process will stop automatically with an error.
1254
+ conf = <<CONF
1255
+ <source>
1256
+ @type plugin_not_found
1257
+ tag test
1258
+ </source>
1259
+ CONF
1260
+ conf_path = create_conf_file('plugin_not_found.conf', conf)
1261
+ assert File.exist?(conf_path)
1262
+ assert_log_matches(create_cmdline(conf_path, option),
1263
+ "[error]",
1264
+ patterns_not_match: ["[warn]"])
1265
+ end
1266
+
1267
+ test 'system config one should not be overwritten when cmd line one is not specified' do
1268
+ conf = <<CONF
1269
+ <system>
1270
+ log_level debug
1271
+ </system>
1272
+ CONF
1273
+ conf_path = create_conf_file('debug.conf', conf)
1274
+ assert File.exist?(conf_path)
1275
+ assert_log_matches(create_cmdline(conf_path),
1276
+ "[debug]")
1277
+ end
1278
+ end
1279
+
1280
+ sub_test_case "inline_config" do
1281
+ test "can change log_level by --inline-config" do
1282
+ # Since we can't define multiple `<system>` directives, this use-case is not recommended.
1283
+ # This is just for this test.
1284
+ inline_conf = '<system>\nlog_level debug\n</system>'
1285
+ conf_path = create_conf_file('test.conf', "")
1286
+ assert File.exist?(conf_path)
1287
+ assert_log_matches(create_cmdline(conf_path, "--inline-config", inline_conf),
1288
+ "[debug]")
1289
+ end
1290
+ end
1106
1291
  end
@@ -188,7 +188,6 @@ slow_flush_log_threshold: float: (20.0)
188
188
  retry_exponential_backoff_base: float: (2)
189
189
  retry_max_interval: time: (nil)
190
190
  retry_randomize: bool: (true)
191
- disable_chunk_backup: bool: (false)
192
191
  <secondary>: optional, single
193
192
  @type: string: (nil)
194
193
  <buffer>: optional, single
@@ -22,7 +22,7 @@ class TextParserTest < ::Test::Unit::TestCase
22
22
  Fluent::TextParser.register_template('multi_event_test', Proc.new { MultiEventTestParser.new })
23
23
 
24
24
  def test_lookup_unknown_format
25
- assert_raise Fluent::ConfigError do
25
+ assert_raise Fluent::NotFoundPluginError do
26
26
  Fluent::Plugin.new_parser('unknown')
27
27
  end
28
28
  end
@@ -38,14 +38,14 @@ class TextParserTest < ::Test::Unit::TestCase
38
38
 
39
39
  def test_parse_with_return
40
40
  parser = Fluent::TextParser.new
41
- parser.configure('format' => 'none')
41
+ parser.configure(config_element('test', '', 'format' => 'none'))
42
42
  _time, record = parser.parse('log message!')
43
43
  assert_equal({'message' => 'log message!'}, record)
44
44
  end
45
45
 
46
46
  def test_parse_with_block
47
47
  parser = Fluent::TextParser.new
48
- parser.configure('format' => 'none')
48
+ parser.configure(config_element('test', '', 'format' => 'none'))
49
49
  parser.parse('log message!') { |time, record|
50
50
  assert_equal({'message' => 'log message!'}, record)
51
51
  }
@@ -53,7 +53,7 @@ class TextParserTest < ::Test::Unit::TestCase
53
53
 
54
54
  def test_multi_event_parser
55
55
  parser = Fluent::TextParser.new
56
- parser.configure('format' => 'multi_event_test')
56
+ parser.configure(config_element('test', '', 'format' => 'multi_event_test'))
57
57
  i = 0
58
58
  parser.parse('log message!') { |time, record|
59
59
  assert_equal('log message!', record['message'])
@@ -67,7 +67,7 @@ class TextParserTest < ::Test::Unit::TestCase
67
67
  assert_nil p1.estimate_current_event
68
68
  assert_nil p1.parser
69
69
 
70
- p1.configure('format' => 'none')
70
+ p1.configure(config_element('test', '', 'format' => 'none'))
71
71
  assert_equal true, p1.parser.estimate_current_event
72
72
 
73
73
  p2 = Fluent::TextParser.new
@@ -76,7 +76,7 @@ class TextParserTest < ::Test::Unit::TestCase
76
76
 
77
77
  p2.estimate_current_event = false
78
78
 
79
- p2.configure('format' => 'none')
79
+ p2.configure(config_element('test', '', 'format' => 'none'))
80
80
  assert_equal false, p2.parser.estimate_current_event
81
81
  end
82
82
 
@@ -5,13 +5,6 @@ require 'fluent/config/section'
5
5
  require 'fluent/system_config'
6
6
 
7
7
  module Fluent::Config
8
- class FakeLoggerInitializer
9
- attr_accessor :level
10
- def initialize
11
- @level = nil
12
- end
13
- end
14
-
15
8
  class FakeSupervisor
16
9
  attr_writer :log_level
17
10
 
@@ -19,8 +12,8 @@ module Fluent::Config
19
12
  @system_config = nil
20
13
  @cl_opt = {
21
14
  wokers: nil,
15
+ restart_worker_interval: nil,
22
16
  root_dir: nil,
23
- log: FakeLoggerInitializer.new,
24
17
  log_level: Fluent::Log::LEVEL_INFO,
25
18
  suppress_interval: nil,
26
19
  suppress_config_dump: nil,
@@ -28,6 +21,8 @@ module Fluent::Config
28
21
  log_event_label: nil,
29
22
  log_event_verbose: nil,
30
23
  without_source: nil,
24
+ enable_input_metrics: nil,
25
+ enable_size_metrics: nil,
31
26
  emit_error_log_interval: nil,
32
27
  file_permission: nil,
33
28
  dir_permission: nil,
@@ -70,6 +65,7 @@ module Fluent::Config
70
65
  sc = Fluent::SystemConfig.new(conf)
71
66
  sc.overwrite_variables(**s.for_system_config)
72
67
  assert_equal(1, sc.workers)
68
+ assert_equal(0, sc.restart_worker_interval)
73
69
  assert_nil(sc.root_dir)
74
70
  assert_equal(Fluent::Log::LEVEL_INFO, sc.log_level)
75
71
  assert_nil(sc.suppress_repeated_stacktrace)
@@ -77,13 +73,17 @@ module Fluent::Config
77
73
  assert_nil(sc.emit_error_log_interval)
78
74
  assert_nil(sc.suppress_config_dump)
79
75
  assert_nil(sc.without_source)
76
+ assert_nil(sc.enable_input_metrics)
77
+ assert_nil(sc.enable_size_metrics)
80
78
  assert_nil(sc.enable_msgpack_time_support)
79
+ assert(!sc.enable_jit)
81
80
  assert_equal(:text, sc.log.format)
82
81
  assert_equal('%Y-%m-%d %H:%M:%S %z', sc.log.time_format)
83
82
  end
84
83
 
85
84
  data(
86
85
  'workers' => ['workers', 3],
86
+ 'restart_worker_interval' => ['restart_worker_interval', 60],
87
87
  'root_dir' => ['root_dir', File.join(TMP_DIR, 'root')],
88
88
  'log_level' => ['log_level', 'error'],
89
89
  'suppress_repeated_stacktrace' => ['suppress_repeated_stacktrace', true],
@@ -93,6 +93,9 @@ module Fluent::Config
93
93
  'without_source' => ['without_source', true],
94
94
  'strict_config_value' => ['strict_config_value', true],
95
95
  'enable_msgpack_time_support' => ['enable_msgpack_time_support', true],
96
+ 'enable_input_metrics' => ['enable_input_metrics', true],
97
+ 'enable_size_metrics' => ['enable_size_metrics', true],
98
+ 'enable_jit' => ['enable_jit', true],
96
99
  )
97
100
  test "accepts parameters" do |(k, v)|
98
101
  conf = parse_text(<<-EOS)
@@ -148,7 +151,7 @@ module Fluent::Config
148
151
  data('daily' => "daily",
149
152
  'weekly' => 'weekly',
150
153
  'monthly' => 'monthly')
151
- test "symbols for rotate_age" do |age|
154
+ test "strings for rotate_age" do |age|
152
155
  conf = parse_text(<<-EOS)
153
156
  <system>
154
157
  <log>
@@ -157,7 +160,7 @@ module Fluent::Config
157
160
  </system>
158
161
  EOS
159
162
  sc = Fluent::SystemConfig.new(conf)
160
- assert_equal(age.to_sym, sc.log.rotate_age)
163
+ assert_equal(age, sc.log.rotate_age)
161
164
  end
162
165
 
163
166
  test "numeric number for rotate age" do
@@ -168,7 +171,6 @@ module Fluent::Config
168
171
  </log>
169
172
  </system>
170
173
  EOS
171
- s = FakeSupervisor.new
172
174
  sc = Fluent::SystemConfig.new(conf)
173
175
  assert_equal(3, sc.log.rotate_age)
174
176
  end
@@ -190,7 +190,7 @@ class TestConfigTypes < ::Test::Unit::TestCase
190
190
  data("val" => [:val, 'val'],
191
191
  "v" => [:v, 'v'],
192
192
  "value" => [:value, 'value'])
193
- test 'enum' do |(expected, val, list)|
193
+ test 'enum' do |(expected, val)|
194
194
  assert_equal expected, Config::ENUM_TYPE.call(val, {list: [:val, :value, :v]})
195
195
  end
196
196
 
@@ -0,0 +1,110 @@
1
+ require_relative '../helper'
2
+
3
+ require 'fluent/log'
4
+ require 'fluent/log/console_adapter'
5
+
6
+ class ConsoleAdapterTest < Test::Unit::TestCase
7
+ def setup
8
+ @timestamp = Time.parse("2023-01-01 15:32:41 +0000")
9
+ @timestamp_str = @timestamp.strftime("%Y-%m-%d %H:%M:%S %z")
10
+ Timecop.freeze(@timestamp)
11
+
12
+ @logdev = Fluent::Test::DummyLogDevice.new
13
+ @logger = ServerEngine::DaemonLogger.new(@logdev)
14
+ @fluent_log = Fluent::Log.new(@logger)
15
+ @console_logger = Fluent::Log::ConsoleAdapter.wrap(@fluent_log)
16
+ end
17
+
18
+ def teardown
19
+ Timecop.return
20
+ end
21
+
22
+ def test_expected_log_levels
23
+ assert_equal({debug: 0, info: 1, warn: 2, error: 3, fatal: 4},
24
+ Console::Logger::LEVELS)
25
+ end
26
+
27
+ data(trace: [Fluent::Log::LEVEL_TRACE, :debug],
28
+ debug: [Fluent::Log::LEVEL_DEBUG, :debug],
29
+ info: [Fluent::Log::LEVEL_INFO, :info],
30
+ warn: [Fluent::Log::LEVEL_WARN, :warn],
31
+ error: [Fluent::Log::LEVEL_ERROR, :error],
32
+ fatal: [Fluent::Log::LEVEL_FATAL, :fatal])
33
+ def test_reflect_log_level(data)
34
+ level, expected = data
35
+ @fluent_log.level = level
36
+ console_logger = Fluent::Log::ConsoleAdapter.wrap(@fluent_log)
37
+ assert_equal(Console::Logger::LEVELS[expected],
38
+ console_logger.level)
39
+ end
40
+
41
+ data(debug: :debug,
42
+ info: :info,
43
+ warn: :warn,
44
+ error: :error,
45
+ fatal: :fatal)
46
+ def test_string_subject(level)
47
+ @console_logger.send(level, "subject")
48
+ assert_equal(["#{@timestamp_str} [#{level}]: 0.0s: subject\n"],
49
+ @logdev.logs)
50
+ end
51
+
52
+ data(debug: :debug,
53
+ info: :info,
54
+ warn: :warn,
55
+ error: :error,
56
+ fatal: :fatal)
57
+ def test_args(level)
58
+ @console_logger.send(level, "subject", 1, 2, 3)
59
+ assert_equal([
60
+ "#{@timestamp_str} [#{level}]: 0.0s: subject\n" +
61
+ " | 1\n" +
62
+ " | 2\n" +
63
+ " | 3\n"
64
+ ],
65
+ @logdev.logs)
66
+ end
67
+
68
+ data(debug: :debug,
69
+ info: :info,
70
+ warn: :warn,
71
+ error: :error,
72
+ fatal: :fatal)
73
+ def test_options(level)
74
+ @console_logger.send(level, "subject", kwarg1: "opt1", kwarg2: "opt2")
75
+ assert_equal([
76
+ "#{@timestamp_str} [#{level}]: 0.0s: subject\n" +
77
+ " | {\"kwarg1\":\"opt1\",\"kwarg2\":\"opt2\"}\n"
78
+ ],
79
+ @logdev.logs)
80
+ end
81
+
82
+ data(debug: :debug,
83
+ info: :info,
84
+ warn: :warn,
85
+ error: :error,
86
+ fatal: :fatal)
87
+ def test_block(level)
88
+ @console_logger.send(level, "subject") { "block message" }
89
+ assert_equal([
90
+ "#{@timestamp_str} [#{level}]: 0.0s: subject\n" +
91
+ " | block message\n"
92
+ ],
93
+ @logdev.logs)
94
+ end
95
+
96
+ data(debug: :debug,
97
+ info: :info,
98
+ warn: :warn,
99
+ error: :error,
100
+ fatal: :fatal)
101
+ def test_multiple_entries(level)
102
+ @console_logger.send(level, "subject1")
103
+ @console_logger.send(level, "line2")
104
+ assert_equal([
105
+ "#{@timestamp_str} [#{level}]: 0.0s: subject1\n",
106
+ "#{@timestamp_str} [#{level}]: 0.0s: line2\n"
107
+ ],
108
+ @logdev.logs)
109
+ end
110
+ end