fluentd 1.13.3 → 1.16.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/ISSUE_TEMPLATE/{bug_report.yaml → bug_report.yml} +2 -0
- data/.github/ISSUE_TEMPLATE/config.yml +2 -2
- data/.github/ISSUE_TEMPLATE/{feature_request.yaml → feature_request.yml} +1 -0
- data/.github/workflows/stale-actions.yml +11 -9
- data/.github/workflows/test.yml +32 -0
- data/CHANGELOG.md +490 -10
- data/CONTRIBUTING.md +2 -2
- data/MAINTAINERS.md +7 -5
- data/README.md +3 -23
- data/Rakefile +1 -1
- data/SECURITY.md +14 -0
- data/fluentd.gemspec +7 -8
- data/lib/fluent/command/cat.rb +13 -3
- data/lib/fluent/command/ctl.rb +6 -3
- data/lib/fluent/command/fluentd.rb +73 -65
- data/lib/fluent/command/plugin_config_formatter.rb +1 -1
- data/lib/fluent/compat/output.rb +9 -6
- data/lib/fluent/config/dsl.rb +1 -1
- data/lib/fluent/config/error.rb +12 -0
- data/lib/fluent/config/literal_parser.rb +2 -2
- data/lib/fluent/config/parser.rb +1 -1
- data/lib/fluent/config/v1_parser.rb +3 -3
- data/lib/fluent/config/yaml_parser/fluent_value.rb +47 -0
- data/lib/fluent/config/yaml_parser/loader.rb +108 -0
- data/lib/fluent/config/yaml_parser/parser.rb +166 -0
- data/lib/fluent/config/yaml_parser/section_builder.rb +107 -0
- data/lib/fluent/config/yaml_parser.rb +56 -0
- data/lib/fluent/config.rb +14 -1
- data/lib/fluent/counter/server.rb +1 -1
- data/lib/fluent/counter/validator.rb +3 -3
- data/lib/fluent/daemon.rb +2 -4
- data/lib/fluent/engine.rb +1 -1
- data/lib/fluent/env.rb +4 -0
- data/lib/fluent/error.rb +3 -0
- data/lib/fluent/event.rb +8 -4
- data/lib/fluent/event_router.rb +47 -2
- data/lib/fluent/file_wrapper.rb +137 -0
- data/lib/fluent/log/console_adapter.rb +66 -0
- data/lib/fluent/log.rb +44 -5
- data/lib/fluent/match.rb +1 -1
- data/lib/fluent/msgpack_factory.rb +6 -1
- data/lib/fluent/oj_options.rb +1 -2
- data/lib/fluent/plugin/bare_output.rb +49 -8
- data/lib/fluent/plugin/base.rb +26 -9
- data/lib/fluent/plugin/buf_file.rb +34 -5
- data/lib/fluent/plugin/buf_file_single.rb +32 -3
- data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
- data/lib/fluent/plugin/buffer.rb +216 -70
- data/lib/fluent/plugin/filter.rb +35 -1
- data/lib/fluent/plugin/filter_record_transformer.rb +1 -1
- data/lib/fluent/plugin/in_forward.rb +2 -2
- data/lib/fluent/plugin/in_http.rb +39 -10
- data/lib/fluent/plugin/in_monitor_agent.rb +4 -2
- data/lib/fluent/plugin/in_sample.rb +1 -1
- data/lib/fluent/plugin/in_syslog.rb +13 -1
- data/lib/fluent/plugin/in_tail/group_watch.rb +204 -0
- data/lib/fluent/plugin/in_tail/position_file.rb +33 -33
- data/lib/fluent/plugin/in_tail.rb +216 -84
- data/lib/fluent/plugin/in_tcp.rb +47 -2
- data/lib/fluent/plugin/input.rb +39 -1
- data/lib/fluent/plugin/metrics.rb +119 -0
- data/lib/fluent/plugin/metrics_local.rb +96 -0
- data/lib/fluent/plugin/multi_output.rb +43 -6
- data/lib/fluent/plugin/out_copy.rb +1 -1
- data/lib/fluent/plugin/out_exec_filter.rb +2 -2
- data/lib/fluent/plugin/out_file.rb +20 -2
- data/lib/fluent/plugin/out_forward/ack_handler.rb +19 -4
- data/lib/fluent/plugin/out_forward/socket_cache.rb +2 -0
- data/lib/fluent/plugin/out_forward.rb +17 -9
- data/lib/fluent/plugin/out_secondary_file.rb +39 -22
- data/lib/fluent/plugin/output.rb +167 -78
- data/lib/fluent/plugin/parser.rb +3 -4
- data/lib/fluent/plugin/parser_apache2.rb +1 -1
- data/lib/fluent/plugin/parser_json.rb +1 -1
- data/lib/fluent/plugin/parser_syslog.rb +1 -1
- data/lib/fluent/plugin/storage_local.rb +3 -5
- data/lib/fluent/plugin.rb +10 -1
- data/lib/fluent/plugin_helper/child_process.rb +3 -0
- data/lib/fluent/plugin_helper/event_emitter.rb +8 -1
- data/lib/fluent/plugin_helper/event_loop.rb +2 -2
- data/lib/fluent/plugin_helper/http_server/server.rb +2 -1
- data/lib/fluent/plugin_helper/metrics.rb +129 -0
- data/lib/fluent/plugin_helper/record_accessor.rb +1 -1
- data/lib/fluent/plugin_helper/retry_state.rb +14 -4
- data/lib/fluent/plugin_helper/server.rb +35 -6
- data/lib/fluent/plugin_helper/service_discovery.rb +2 -2
- data/lib/fluent/plugin_helper/socket.rb +13 -2
- data/lib/fluent/plugin_helper/thread.rb +3 -3
- data/lib/fluent/plugin_helper.rb +1 -0
- data/lib/fluent/plugin_id.rb +3 -2
- data/lib/fluent/registry.rb +2 -1
- data/lib/fluent/root_agent.rb +6 -0
- data/lib/fluent/rpc.rb +4 -3
- data/lib/fluent/supervisor.rb +283 -259
- data/lib/fluent/system_config.rb +13 -3
- data/lib/fluent/test/driver/base.rb +11 -5
- data/lib/fluent/test/driver/filter.rb +4 -0
- data/lib/fluent/test/startup_shutdown.rb +6 -8
- data/lib/fluent/time.rb +21 -20
- data/lib/fluent/version.rb +1 -1
- data/lib/fluent/win32api.rb +38 -0
- data/lib/fluent/winsvc.rb +5 -8
- data/templates/new_gem/test/helper.rb.erb +0 -1
- data/test/command/test_cat.rb +31 -2
- data/test/command/test_ctl.rb +1 -2
- data/test/command/test_fluentd.rb +209 -24
- data/test/command/test_plugin_config_formatter.rb +0 -1
- data/test/compat/test_parser.rb +6 -6
- data/test/config/test_system_config.rb +13 -11
- data/test/config/test_types.rb +1 -1
- data/test/log/test_console_adapter.rb +110 -0
- data/test/plugin/in_tail/test_io_handler.rb +26 -8
- data/test/plugin/in_tail/test_position_file.rb +48 -59
- data/test/plugin/out_forward/test_ack_handler.rb +39 -0
- data/test/plugin/out_forward/test_socket_cache.rb +26 -1
- data/test/plugin/test_bare_output.rb +14 -1
- data/test/plugin/test_base.rb +133 -1
- data/test/plugin/test_buf_file.rb +62 -23
- data/test/plugin/test_buf_file_single.rb +65 -0
- data/test/plugin/test_buffer.rb +267 -3
- data/test/plugin/test_buffer_chunk.rb +11 -0
- data/test/plugin/test_filter.rb +12 -1
- data/test/plugin/test_filter_parser.rb +1 -1
- data/test/plugin/test_filter_stdout.rb +2 -2
- data/test/plugin/test_in_forward.rb +9 -11
- data/test/plugin/test_in_http.rb +65 -3
- data/test/plugin/test_in_monitor_agent.rb +216 -11
- data/test/plugin/test_in_object_space.rb +9 -3
- data/test/plugin/test_in_syslog.rb +35 -0
- data/test/plugin/test_in_tail.rb +1393 -385
- data/test/plugin/test_in_tcp.rb +87 -2
- data/test/plugin/test_in_udp.rb +28 -0
- data/test/plugin/test_in_unix.rb +2 -2
- data/test/plugin/test_input.rb +12 -1
- data/test/plugin/test_metrics.rb +294 -0
- data/test/plugin/test_metrics_local.rb +96 -0
- data/test/plugin/test_multi_output.rb +25 -1
- data/test/plugin/test_out_exec.rb +6 -4
- data/test/plugin/test_out_exec_filter.rb +6 -2
- data/test/plugin/test_out_file.rb +34 -17
- data/test/plugin/test_out_forward.rb +78 -77
- data/test/plugin/test_out_http.rb +1 -0
- data/test/plugin/test_out_stdout.rb +2 -2
- data/test/plugin/test_output.rb +297 -12
- data/test/plugin/test_output_as_buffered.rb +44 -44
- data/test/plugin/test_output_as_buffered_compress.rb +32 -18
- data/test/plugin/test_output_as_buffered_retries.rb +54 -7
- data/test/plugin/test_output_as_buffered_secondary.rb +4 -4
- data/test/plugin/test_parser_regexp.rb +1 -6
- data/test/plugin/test_parser_syslog.rb +1 -1
- data/test/plugin_helper/test_cert_option.rb +1 -1
- data/test/plugin_helper/test_child_process.rb +38 -16
- data/test/plugin_helper/test_event_emitter.rb +29 -0
- data/test/plugin_helper/test_http_server_helper.rb +1 -1
- data/test/plugin_helper/test_metrics.rb +137 -0
- data/test/plugin_helper/test_retry_state.rb +602 -38
- data/test/plugin_helper/test_server.rb +78 -6
- data/test/plugin_helper/test_timer.rb +2 -2
- data/test/test_config.rb +191 -24
- data/test/test_event_router.rb +17 -0
- data/test/test_file_wrapper.rb +53 -0
- data/test/test_formatter.rb +24 -21
- data/test/test_log.rb +122 -40
- data/test/test_msgpack_factory.rb +32 -0
- data/test/test_plugin_classes.rb +102 -0
- data/test/test_root_agent.rb +30 -1
- data/test/test_supervisor.rb +477 -257
- data/test/test_time_parser.rb +22 -0
- metadata +55 -34
- data/.drone.yml +0 -35
- data/.github/workflows/issue-auto-closer.yml +0 -12
- data/.github/workflows/linux-test.yaml +0 -36
- data/.github/workflows/macos-test.yaml +0 -30
- data/.github/workflows/windows-test.yaml +0 -46
- data/.gitlab-ci.yml +0 -103
- data/lib/fluent/plugin/file_wrapper.rb +0 -187
- data/test/plugin/test_file_wrapper.rb +0 -126
- data/test/test_logger_initializer.rb +0 -46
@@ -5,20 +5,35 @@ require_relative '../helper'
|
|
5
5
|
|
6
6
|
require 'fileutils'
|
7
7
|
require 'timeout'
|
8
|
+
require 'securerandom'
|
9
|
+
require 'fluent/file_wrapper'
|
8
10
|
|
9
11
|
class TestFluentdCommand < ::Test::Unit::TestCase
|
10
|
-
TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/command/fluentd#{ENV['TEST_ENV_NUMBER']}")
|
11
12
|
SUPERVISOR_PID_PATTERN = /starting fluentd-[.0-9]+ pid=(\d+)/
|
12
13
|
WORKER_PID_PATTERN = /starting fluentd worker pid=(\d+) /
|
13
14
|
|
15
|
+
def tmp_dir
|
16
|
+
File.join(File.dirname(__FILE__), "..", "tmp", "command" "fluentd#{ENV['TEST_ENV_NUMBER']}", SecureRandom.hex(10))
|
17
|
+
end
|
18
|
+
|
14
19
|
setup do
|
15
|
-
|
16
|
-
FileUtils.mkdir_p(
|
20
|
+
@tmp_dir = tmp_dir
|
21
|
+
FileUtils.mkdir_p(@tmp_dir)
|
17
22
|
@supervisor_pid = nil
|
18
23
|
@worker_pids = []
|
19
24
|
ENV["TEST_RUBY_PATH"] = nil
|
20
25
|
end
|
21
26
|
|
27
|
+
teardown do
|
28
|
+
begin
|
29
|
+
FileUtils.rm_rf(@tmp_dir)
|
30
|
+
rescue Errno::EACCES
|
31
|
+
# It may occur on Windows because of delete pending state due to delayed GC.
|
32
|
+
# Ruby 3.2 or later doesn't ignore Errno::EACCES:
|
33
|
+
# https://github.com/ruby/ruby/commit/983115cf3c8f75b1afbe3274f02c1529e1ce3a81
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
22
37
|
def process_exist?(pid)
|
23
38
|
begin
|
24
39
|
r = Process.waitpid(pid, Process::WNOHANG)
|
@@ -30,17 +45,17 @@ class TestFluentdCommand < ::Test::Unit::TestCase
|
|
30
45
|
end
|
31
46
|
|
32
47
|
def create_conf_file(name, content, ext_enc = 'utf-8')
|
33
|
-
conf_path = File.join(
|
34
|
-
|
48
|
+
conf_path = File.join(@tmp_dir, name)
|
49
|
+
Fluent::FileWrapper.open(conf_path, "w:#{ext_enc}:utf-8") do |file|
|
35
50
|
file.write content
|
36
51
|
end
|
37
52
|
conf_path
|
38
53
|
end
|
39
54
|
|
40
55
|
def create_plugin_file(name, content)
|
41
|
-
file_path = File.join(
|
56
|
+
file_path = File.join(@tmp_dir, 'plugin', name)
|
42
57
|
FileUtils.mkdir_p(File.dirname(file_path))
|
43
|
-
|
58
|
+
Fluent::FileWrapper.open(file_path, 'w') do |file|
|
44
59
|
file.write content
|
45
60
|
end
|
46
61
|
file_path
|
@@ -56,8 +71,22 @@ class TestFluentdCommand < ::Test::Unit::TestCase
|
|
56
71
|
end
|
57
72
|
end
|
58
73
|
|
59
|
-
def
|
60
|
-
|
74
|
+
def process_kill(pid)
|
75
|
+
if Fluent.windows?
|
76
|
+
Process.kill(:KILL, pid) rescue nil
|
77
|
+
return
|
78
|
+
end
|
79
|
+
|
80
|
+
begin
|
81
|
+
Process.kill(:TERM, pid) rescue nil
|
82
|
+
Timeout.timeout(10){ sleep 0.1 while process_exist?(pid) }
|
83
|
+
rescue Timeout::Error
|
84
|
+
Process.kill(:KILL, pid) rescue nil
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
def execute_command(cmdline, chdir=@tmp_dir, env = {})
|
89
|
+
null_stream = Fluent::FileWrapper.open(File::NULL, 'w')
|
61
90
|
gemfile_path = File.expand_path(File.dirname(__FILE__) + "../../../Gemfile")
|
62
91
|
|
63
92
|
env = { "BUNDLE_GEMFILE" => gemfile_path }.merge(env)
|
@@ -70,12 +99,12 @@ class TestFluentdCommand < ::Test::Unit::TestCase
|
|
70
99
|
yield pid, io
|
71
100
|
# p(here: "execute command", pid: pid, worker_pids: @worker_pids)
|
72
101
|
ensure
|
73
|
-
|
102
|
+
process_kill(pid)
|
74
103
|
if @supervisor_pid
|
75
|
-
|
104
|
+
process_kill(@supervisor_pid)
|
76
105
|
end
|
77
106
|
@worker_pids.each do |cpid|
|
78
|
-
|
107
|
+
process_kill(cpid)
|
79
108
|
end
|
80
109
|
# p(here: "execute command", pid: pid, exist: process_exist?(pid), worker_pids: @worker_pids, exists: @worker_pids.map{|i| process_exist?(i) })
|
81
110
|
Timeout.timeout(10){ sleep 0.1 while process_exist?(pid) }
|
@@ -97,16 +126,18 @@ class TestFluentdCommand < ::Test::Unit::TestCase
|
|
97
126
|
end
|
98
127
|
end
|
99
128
|
|
100
|
-
|
129
|
+
# ATTENTION: This stops taking logs when all `pattern_list` match or timeout,
|
130
|
+
# so `patterns_not_match` can test only logs up to that point.
|
131
|
+
def assert_log_matches(cmdline, *pattern_list, patterns_not_match: [], timeout: 20, env: {})
|
101
132
|
matched = false
|
102
133
|
matched_wrongly = false
|
103
134
|
assert_error_msg = ""
|
104
135
|
stdio_buf = ""
|
105
136
|
begin
|
106
|
-
execute_command(cmdline,
|
137
|
+
execute_command(cmdline, @tmp_dir, env) do |pid, stdout|
|
107
138
|
begin
|
108
139
|
waiting(timeout) do
|
109
|
-
while process_exist?(pid)
|
140
|
+
while process_exist?(pid)
|
110
141
|
readables, _, _ = IO.select([stdout], nil, nil, 1)
|
111
142
|
next unless readables
|
112
143
|
break if readables.first.eof?
|
@@ -118,6 +149,18 @@ class TestFluentdCommand < ::Test::Unit::TestCase
|
|
118
149
|
if pattern_list.all?{|ptn| lines.any?{|line| ptn.is_a?(Regexp) ? ptn.match(line) : line.include?(ptn) } }
|
119
150
|
matched = true
|
120
151
|
end
|
152
|
+
|
153
|
+
if Fluent.windows?
|
154
|
+
# https://github.com/fluent/fluentd/issues/4095
|
155
|
+
# On Windows, the initial process is different from the supervisor process,
|
156
|
+
# so we need to wait until `SUPERVISOR_PID_PATTERN` appears in the logs to get the pid.
|
157
|
+
# (Worker processes will be killed by the supervisor process, so we don't need it-)
|
158
|
+
break if matched && SUPERVISOR_PID_PATTERN =~ stdio_buf
|
159
|
+
else
|
160
|
+
# On Non-Windows, the initial process is the supervisor process,
|
161
|
+
# so we don't need to wait `SUPERVISOR_PID_PATTERN`.
|
162
|
+
break if matched
|
163
|
+
end
|
121
164
|
end
|
122
165
|
end
|
123
166
|
ensure
|
@@ -131,6 +174,10 @@ class TestFluentdCommand < ::Test::Unit::TestCase
|
|
131
174
|
end
|
132
175
|
rescue Timeout::Error
|
133
176
|
assert_error_msg = "execution timeout"
|
177
|
+
# https://github.com/fluent/fluentd/issues/4095
|
178
|
+
# On Windows, timeout without `@supervisor_pid` means that the test is invalid,
|
179
|
+
# since the supervisor process will survive without being killed correctly.
|
180
|
+
flunk("Invalid test: The pid of supervisor could not be taken, which is necessary on Windows.") if Fluent.windows? && @supervisor_pid.nil?
|
134
181
|
rescue => e
|
135
182
|
assert_error_msg = "unexpected error in launching fluentd: #{e.inspect}"
|
136
183
|
else
|
@@ -162,7 +209,7 @@ class TestFluentdCommand < ::Test::Unit::TestCase
|
|
162
209
|
assert matched && !matched_wrongly, assert_error_msg
|
163
210
|
end
|
164
211
|
|
165
|
-
def assert_fluentd_fails_to_start(cmdline, *pattern_list, timeout:
|
212
|
+
def assert_fluentd_fails_to_start(cmdline, *pattern_list, timeout: 20)
|
166
213
|
# empty_list.all?{ ... } is always true
|
167
214
|
matched = false
|
168
215
|
running = false
|
@@ -198,6 +245,10 @@ class TestFluentdCommand < ::Test::Unit::TestCase
|
|
198
245
|
end
|
199
246
|
rescue Timeout::Error
|
200
247
|
assert_error_msg = "execution timeout with command out:\n" + stdio_buf
|
248
|
+
# https://github.com/fluent/fluentd/issues/4095
|
249
|
+
# On Windows, timeout without `@supervisor_pid` means that the test is invalid,
|
250
|
+
# since the supervisor process will survive without being killed correctly.
|
251
|
+
flunk("Invalid test: The pid of supervisor could not be taken, which is necessary on Windows.") if Fluent.windows? && @supervisor_pid.nil?
|
201
252
|
rescue => e
|
202
253
|
assert_error_msg = "unexpected error in launching fluentd: #{e.inspect}\n" + stdio_buf
|
203
254
|
assert false, assert_error_msg
|
@@ -269,7 +320,7 @@ CONF
|
|
269
320
|
|
270
321
|
sub_test_case 'with system configuration about root directory' do
|
271
322
|
setup do
|
272
|
-
@root_path = File.join(
|
323
|
+
@root_path = File.join(@tmp_dir, "rootpath")
|
273
324
|
FileUtils.rm_rf(@root_path)
|
274
325
|
@conf = <<CONF
|
275
326
|
<system>
|
@@ -308,7 +359,7 @@ CONF
|
|
308
359
|
end
|
309
360
|
|
310
361
|
test 'fails to launch fluentd if specified root path is invalid path for directory' do
|
311
|
-
|
362
|
+
Fluent::FileWrapper.open(@root_path, 'w') do |_|
|
312
363
|
# create file and close it
|
313
364
|
end
|
314
365
|
conf_path = create_conf_file('existing_root_dir.conf', @conf)
|
@@ -508,7 +559,7 @@ CONF
|
|
508
559
|
|
509
560
|
assert_fluentd_fails_to_start(
|
510
561
|
create_cmdline(conf_path, "-p", File.dirname(plugin_path)),
|
511
|
-
"in_buggy.rb:5: syntax error, unexpected end-of-input
|
562
|
+
"in_buggy.rb:5: syntax error, unexpected end-of-input"
|
512
563
|
)
|
513
564
|
end
|
514
565
|
end
|
@@ -554,7 +605,7 @@ CONF
|
|
554
605
|
|
555
606
|
sub_test_case 'configured to run 2 workers' do
|
556
607
|
setup do
|
557
|
-
@root_path = File.join(
|
608
|
+
@root_path = File.join(@tmp_dir, "rootpath")
|
558
609
|
FileUtils.rm_rf(@root_path)
|
559
610
|
FileUtils.mkdir_p(@root_path)
|
560
611
|
end
|
@@ -589,6 +640,39 @@ CONF
|
|
589
640
|
)
|
590
641
|
end
|
591
642
|
|
643
|
+
sub_test_case "YAML config format" do
|
644
|
+
test 'success to start the number of workers specified in configuration' do
|
645
|
+
conf = <<'CONF'
|
646
|
+
system:
|
647
|
+
workers: 2
|
648
|
+
root_dir: "#{@root_path}"
|
649
|
+
config:
|
650
|
+
- source:
|
651
|
+
$type: dummy
|
652
|
+
$id: !fluent/s "dummy.#{worker_id}" # check worker_id works or not with actual command
|
653
|
+
$label: '@dummydata'
|
654
|
+
tag: dummy
|
655
|
+
dummy: !fluent/json {"message": !fluent/s "yay from #{hostname}!"}
|
656
|
+
|
657
|
+
- label:
|
658
|
+
$name: '@dummydata'
|
659
|
+
config:
|
660
|
+
- match:
|
661
|
+
$tag: dummy
|
662
|
+
$type: "null"
|
663
|
+
$id: blackhole
|
664
|
+
CONF
|
665
|
+
conf_path = create_conf_file('workers1.yaml', conf)
|
666
|
+
assert Dir.exist?(@root_path)
|
667
|
+
|
668
|
+
assert_log_matches(
|
669
|
+
create_cmdline(conf_path),
|
670
|
+
"#0 fluentd worker is now running worker=0",
|
671
|
+
"#1 fluentd worker is now running worker=1"
|
672
|
+
)
|
673
|
+
end
|
674
|
+
end
|
675
|
+
|
592
676
|
test 'success to start the number of workers specified by command line option' do
|
593
677
|
conf = <<CONF
|
594
678
|
<system>
|
@@ -857,7 +941,7 @@ CONF
|
|
857
941
|
'-external-encoding' => '--external-encoding=utf-8',
|
858
942
|
'-internal-encoding' => '--internal-encoding=utf-8',
|
859
943
|
)
|
860
|
-
test "-E option is set to RUBYOPT" do |
|
944
|
+
test "-E option is set to RUBYOPT" do |base_opt|
|
861
945
|
conf = <<CONF
|
862
946
|
<source>
|
863
947
|
@type dummy
|
@@ -868,6 +952,7 @@ CONF
|
|
868
952
|
</match>
|
869
953
|
CONF
|
870
954
|
conf_path = create_conf_file('rubyopt_test.conf', conf)
|
955
|
+
opt = base_opt.dup
|
871
956
|
opt << " #{ENV['RUBYOPT']}" if ENV['RUBYOPT']
|
872
957
|
assert_log_matches(
|
873
958
|
create_cmdline(conf_path),
|
@@ -907,9 +992,14 @@ CONF
|
|
907
992
|
</match>
|
908
993
|
CONF
|
909
994
|
conf_path = create_conf_file('rubyopt_invalid_test.conf', conf)
|
995
|
+
if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('3.3.0')
|
996
|
+
expected_phrase = 'ruby: invalid switch in RUBYOPT'
|
997
|
+
else
|
998
|
+
expected_phrase = 'Invalid option is passed to RUBYOPT'
|
999
|
+
end
|
910
1000
|
assert_log_matches(
|
911
1001
|
create_cmdline(conf_path),
|
912
|
-
|
1002
|
+
expected_phrase,
|
913
1003
|
env: { 'RUBYOPT' => 'a' },
|
914
1004
|
)
|
915
1005
|
end
|
@@ -928,10 +1018,10 @@ CONF
|
|
928
1018
|
</match>
|
929
1019
|
CONF
|
930
1020
|
ruby_path = ServerEngine.ruby_bin_path
|
931
|
-
tmp_ruby_path = File.join(
|
1021
|
+
tmp_ruby_path = File.join(@tmp_dir, "ruby with spaces")
|
932
1022
|
if Fluent.windows?
|
933
1023
|
tmp_ruby_path << ".bat"
|
934
|
-
|
1024
|
+
Fluent::FileWrapper.open(tmp_ruby_path, "w") do |file|
|
935
1025
|
file.write "#{ruby_path} %*"
|
936
1026
|
end
|
937
1027
|
else
|
@@ -1103,4 +1193,99 @@ CONF
|
|
1103
1193
|
"shared socket for multiple workers is disabled",)
|
1104
1194
|
end
|
1105
1195
|
end
|
1196
|
+
|
1197
|
+
# TODO: `patterns_not_match` can test only logs up to `pattern_list`,
|
1198
|
+
# so we need to fix some meaningless `patterns_not_match` conditions.
|
1199
|
+
sub_test_case 'log_level by command line option' do
|
1200
|
+
test 'info' do
|
1201
|
+
conf = ""
|
1202
|
+
conf_path = create_conf_file('empty.conf', conf)
|
1203
|
+
assert File.exist?(conf_path)
|
1204
|
+
assert_log_matches(create_cmdline(conf_path),
|
1205
|
+
"[info]",
|
1206
|
+
patterns_not_match: ["[debug]"])
|
1207
|
+
end
|
1208
|
+
|
1209
|
+
test 'debug' do
|
1210
|
+
conf = ""
|
1211
|
+
conf_path = create_conf_file('empty.conf', conf)
|
1212
|
+
assert File.exist?(conf_path)
|
1213
|
+
assert_log_matches(create_cmdline(conf_path, "-v"),
|
1214
|
+
"[debug]",
|
1215
|
+
patterns_not_match: ["[trace]"])
|
1216
|
+
end
|
1217
|
+
|
1218
|
+
data("Trace" => "-vv")
|
1219
|
+
data("Invalid low level should be treated as Trace level": "-vvv")
|
1220
|
+
test 'trace' do |option|
|
1221
|
+
conf = <<CONF
|
1222
|
+
<source>
|
1223
|
+
@type sample
|
1224
|
+
tag test
|
1225
|
+
</source>
|
1226
|
+
CONF
|
1227
|
+
conf_path = create_conf_file('sample.conf', conf)
|
1228
|
+
assert File.exist?(conf_path)
|
1229
|
+
assert_log_matches(create_cmdline(conf_path, option),
|
1230
|
+
"[trace]",)
|
1231
|
+
end
|
1232
|
+
|
1233
|
+
test 'warn' do
|
1234
|
+
omit "Can't run on Windows since there is no way to take pid of the supervisor." if Fluent.windows?
|
1235
|
+
conf = <<CONF
|
1236
|
+
<source>
|
1237
|
+
@type sample
|
1238
|
+
tag test
|
1239
|
+
</source>
|
1240
|
+
CONF
|
1241
|
+
conf_path = create_conf_file('sample.conf', conf)
|
1242
|
+
assert File.exist?(conf_path)
|
1243
|
+
assert_log_matches(create_cmdline(conf_path, "-q"),
|
1244
|
+
"[warn]",
|
1245
|
+
patterns_not_match: ["[info]"])
|
1246
|
+
end
|
1247
|
+
|
1248
|
+
data("Error" => "-qq")
|
1249
|
+
data("Fatal should be treated as Error level" => "-qqq")
|
1250
|
+
data("Invalid high level should be treated as Error level": "-qqqq")
|
1251
|
+
test 'error' do |option|
|
1252
|
+
# This test can run on Windows correctly,
|
1253
|
+
# since the process will stop automatically with an error.
|
1254
|
+
conf = <<CONF
|
1255
|
+
<source>
|
1256
|
+
@type plugin_not_found
|
1257
|
+
tag test
|
1258
|
+
</source>
|
1259
|
+
CONF
|
1260
|
+
conf_path = create_conf_file('plugin_not_found.conf', conf)
|
1261
|
+
assert File.exist?(conf_path)
|
1262
|
+
assert_log_matches(create_cmdline(conf_path, option),
|
1263
|
+
"[error]",
|
1264
|
+
patterns_not_match: ["[warn]"])
|
1265
|
+
end
|
1266
|
+
|
1267
|
+
test 'system config one should not be overwritten when cmd line one is not specified' do
|
1268
|
+
conf = <<CONF
|
1269
|
+
<system>
|
1270
|
+
log_level debug
|
1271
|
+
</system>
|
1272
|
+
CONF
|
1273
|
+
conf_path = create_conf_file('debug.conf', conf)
|
1274
|
+
assert File.exist?(conf_path)
|
1275
|
+
assert_log_matches(create_cmdline(conf_path),
|
1276
|
+
"[debug]")
|
1277
|
+
end
|
1278
|
+
end
|
1279
|
+
|
1280
|
+
sub_test_case "inline_config" do
|
1281
|
+
test "can change log_level by --inline-config" do
|
1282
|
+
# Since we can't define multiple `<system>` directives, this use-case is not recommended.
|
1283
|
+
# This is just for this test.
|
1284
|
+
inline_conf = '<system>\nlog_level debug\n</system>'
|
1285
|
+
conf_path = create_conf_file('test.conf', "")
|
1286
|
+
assert File.exist?(conf_path)
|
1287
|
+
assert_log_matches(create_cmdline(conf_path, "--inline-config", inline_conf),
|
1288
|
+
"[debug]")
|
1289
|
+
end
|
1290
|
+
end
|
1106
1291
|
end
|
@@ -188,7 +188,6 @@ slow_flush_log_threshold: float: (20.0)
|
|
188
188
|
retry_exponential_backoff_base: float: (2)
|
189
189
|
retry_max_interval: time: (nil)
|
190
190
|
retry_randomize: bool: (true)
|
191
|
-
disable_chunk_backup: bool: (false)
|
192
191
|
<secondary>: optional, single
|
193
192
|
@type: string: (nil)
|
194
193
|
<buffer>: optional, single
|
data/test/compat/test_parser.rb
CHANGED
@@ -22,7 +22,7 @@ class TextParserTest < ::Test::Unit::TestCase
|
|
22
22
|
Fluent::TextParser.register_template('multi_event_test', Proc.new { MultiEventTestParser.new })
|
23
23
|
|
24
24
|
def test_lookup_unknown_format
|
25
|
-
assert_raise Fluent::
|
25
|
+
assert_raise Fluent::NotFoundPluginError do
|
26
26
|
Fluent::Plugin.new_parser('unknown')
|
27
27
|
end
|
28
28
|
end
|
@@ -38,14 +38,14 @@ class TextParserTest < ::Test::Unit::TestCase
|
|
38
38
|
|
39
39
|
def test_parse_with_return
|
40
40
|
parser = Fluent::TextParser.new
|
41
|
-
parser.configure('format' => 'none')
|
41
|
+
parser.configure(config_element('test', '', 'format' => 'none'))
|
42
42
|
_time, record = parser.parse('log message!')
|
43
43
|
assert_equal({'message' => 'log message!'}, record)
|
44
44
|
end
|
45
45
|
|
46
46
|
def test_parse_with_block
|
47
47
|
parser = Fluent::TextParser.new
|
48
|
-
parser.configure('format' => 'none')
|
48
|
+
parser.configure(config_element('test', '', 'format' => 'none'))
|
49
49
|
parser.parse('log message!') { |time, record|
|
50
50
|
assert_equal({'message' => 'log message!'}, record)
|
51
51
|
}
|
@@ -53,7 +53,7 @@ class TextParserTest < ::Test::Unit::TestCase
|
|
53
53
|
|
54
54
|
def test_multi_event_parser
|
55
55
|
parser = Fluent::TextParser.new
|
56
|
-
parser.configure('format' => 'multi_event_test')
|
56
|
+
parser.configure(config_element('test', '', 'format' => 'multi_event_test'))
|
57
57
|
i = 0
|
58
58
|
parser.parse('log message!') { |time, record|
|
59
59
|
assert_equal('log message!', record['message'])
|
@@ -67,7 +67,7 @@ class TextParserTest < ::Test::Unit::TestCase
|
|
67
67
|
assert_nil p1.estimate_current_event
|
68
68
|
assert_nil p1.parser
|
69
69
|
|
70
|
-
p1.configure('format' => 'none')
|
70
|
+
p1.configure(config_element('test', '', 'format' => 'none'))
|
71
71
|
assert_equal true, p1.parser.estimate_current_event
|
72
72
|
|
73
73
|
p2 = Fluent::TextParser.new
|
@@ -76,7 +76,7 @@ class TextParserTest < ::Test::Unit::TestCase
|
|
76
76
|
|
77
77
|
p2.estimate_current_event = false
|
78
78
|
|
79
|
-
p2.configure('format' => 'none')
|
79
|
+
p2.configure(config_element('test', '', 'format' => 'none'))
|
80
80
|
assert_equal false, p2.parser.estimate_current_event
|
81
81
|
end
|
82
82
|
|
@@ -5,13 +5,6 @@ require 'fluent/config/section'
|
|
5
5
|
require 'fluent/system_config'
|
6
6
|
|
7
7
|
module Fluent::Config
|
8
|
-
class FakeLoggerInitializer
|
9
|
-
attr_accessor :level
|
10
|
-
def initialize
|
11
|
-
@level = nil
|
12
|
-
end
|
13
|
-
end
|
14
|
-
|
15
8
|
class FakeSupervisor
|
16
9
|
attr_writer :log_level
|
17
10
|
|
@@ -19,8 +12,8 @@ module Fluent::Config
|
|
19
12
|
@system_config = nil
|
20
13
|
@cl_opt = {
|
21
14
|
wokers: nil,
|
15
|
+
restart_worker_interval: nil,
|
22
16
|
root_dir: nil,
|
23
|
-
log: FakeLoggerInitializer.new,
|
24
17
|
log_level: Fluent::Log::LEVEL_INFO,
|
25
18
|
suppress_interval: nil,
|
26
19
|
suppress_config_dump: nil,
|
@@ -28,6 +21,8 @@ module Fluent::Config
|
|
28
21
|
log_event_label: nil,
|
29
22
|
log_event_verbose: nil,
|
30
23
|
without_source: nil,
|
24
|
+
enable_input_metrics: nil,
|
25
|
+
enable_size_metrics: nil,
|
31
26
|
emit_error_log_interval: nil,
|
32
27
|
file_permission: nil,
|
33
28
|
dir_permission: nil,
|
@@ -70,6 +65,7 @@ module Fluent::Config
|
|
70
65
|
sc = Fluent::SystemConfig.new(conf)
|
71
66
|
sc.overwrite_variables(**s.for_system_config)
|
72
67
|
assert_equal(1, sc.workers)
|
68
|
+
assert_equal(0, sc.restart_worker_interval)
|
73
69
|
assert_nil(sc.root_dir)
|
74
70
|
assert_equal(Fluent::Log::LEVEL_INFO, sc.log_level)
|
75
71
|
assert_nil(sc.suppress_repeated_stacktrace)
|
@@ -77,13 +73,17 @@ module Fluent::Config
|
|
77
73
|
assert_nil(sc.emit_error_log_interval)
|
78
74
|
assert_nil(sc.suppress_config_dump)
|
79
75
|
assert_nil(sc.without_source)
|
76
|
+
assert_nil(sc.enable_input_metrics)
|
77
|
+
assert_nil(sc.enable_size_metrics)
|
80
78
|
assert_nil(sc.enable_msgpack_time_support)
|
79
|
+
assert(!sc.enable_jit)
|
81
80
|
assert_equal(:text, sc.log.format)
|
82
81
|
assert_equal('%Y-%m-%d %H:%M:%S %z', sc.log.time_format)
|
83
82
|
end
|
84
83
|
|
85
84
|
data(
|
86
85
|
'workers' => ['workers', 3],
|
86
|
+
'restart_worker_interval' => ['restart_worker_interval', 60],
|
87
87
|
'root_dir' => ['root_dir', File.join(TMP_DIR, 'root')],
|
88
88
|
'log_level' => ['log_level', 'error'],
|
89
89
|
'suppress_repeated_stacktrace' => ['suppress_repeated_stacktrace', true],
|
@@ -93,6 +93,9 @@ module Fluent::Config
|
|
93
93
|
'without_source' => ['without_source', true],
|
94
94
|
'strict_config_value' => ['strict_config_value', true],
|
95
95
|
'enable_msgpack_time_support' => ['enable_msgpack_time_support', true],
|
96
|
+
'enable_input_metrics' => ['enable_input_metrics', true],
|
97
|
+
'enable_size_metrics' => ['enable_size_metrics', true],
|
98
|
+
'enable_jit' => ['enable_jit', true],
|
96
99
|
)
|
97
100
|
test "accepts parameters" do |(k, v)|
|
98
101
|
conf = parse_text(<<-EOS)
|
@@ -148,7 +151,7 @@ module Fluent::Config
|
|
148
151
|
data('daily' => "daily",
|
149
152
|
'weekly' => 'weekly',
|
150
153
|
'monthly' => 'monthly')
|
151
|
-
test "
|
154
|
+
test "strings for rotate_age" do |age|
|
152
155
|
conf = parse_text(<<-EOS)
|
153
156
|
<system>
|
154
157
|
<log>
|
@@ -157,7 +160,7 @@ module Fluent::Config
|
|
157
160
|
</system>
|
158
161
|
EOS
|
159
162
|
sc = Fluent::SystemConfig.new(conf)
|
160
|
-
assert_equal(age
|
163
|
+
assert_equal(age, sc.log.rotate_age)
|
161
164
|
end
|
162
165
|
|
163
166
|
test "numeric number for rotate age" do
|
@@ -168,7 +171,6 @@ module Fluent::Config
|
|
168
171
|
</log>
|
169
172
|
</system>
|
170
173
|
EOS
|
171
|
-
s = FakeSupervisor.new
|
172
174
|
sc = Fluent::SystemConfig.new(conf)
|
173
175
|
assert_equal(3, sc.log.rotate_age)
|
174
176
|
end
|
data/test/config/test_types.rb
CHANGED
@@ -190,7 +190,7 @@ class TestConfigTypes < ::Test::Unit::TestCase
|
|
190
190
|
data("val" => [:val, 'val'],
|
191
191
|
"v" => [:v, 'v'],
|
192
192
|
"value" => [:value, 'value'])
|
193
|
-
test 'enum' do |(expected, val
|
193
|
+
test 'enum' do |(expected, val)|
|
194
194
|
assert_equal expected, Config::ENUM_TYPE.call(val, {list: [:val, :value, :v]})
|
195
195
|
end
|
196
196
|
|
@@ -0,0 +1,110 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
|
3
|
+
require 'fluent/log'
|
4
|
+
require 'fluent/log/console_adapter'
|
5
|
+
|
6
|
+
class ConsoleAdapterTest < Test::Unit::TestCase
|
7
|
+
def setup
|
8
|
+
@timestamp = Time.parse("2023-01-01 15:32:41 +0000")
|
9
|
+
@timestamp_str = @timestamp.strftime("%Y-%m-%d %H:%M:%S %z")
|
10
|
+
Timecop.freeze(@timestamp)
|
11
|
+
|
12
|
+
@logdev = Fluent::Test::DummyLogDevice.new
|
13
|
+
@logger = ServerEngine::DaemonLogger.new(@logdev)
|
14
|
+
@fluent_log = Fluent::Log.new(@logger)
|
15
|
+
@console_logger = Fluent::Log::ConsoleAdapter.wrap(@fluent_log)
|
16
|
+
end
|
17
|
+
|
18
|
+
def teardown
|
19
|
+
Timecop.return
|
20
|
+
end
|
21
|
+
|
22
|
+
def test_expected_log_levels
|
23
|
+
assert_equal({debug: 0, info: 1, warn: 2, error: 3, fatal: 4},
|
24
|
+
Console::Logger::LEVELS)
|
25
|
+
end
|
26
|
+
|
27
|
+
data(trace: [Fluent::Log::LEVEL_TRACE, :debug],
|
28
|
+
debug: [Fluent::Log::LEVEL_DEBUG, :debug],
|
29
|
+
info: [Fluent::Log::LEVEL_INFO, :info],
|
30
|
+
warn: [Fluent::Log::LEVEL_WARN, :warn],
|
31
|
+
error: [Fluent::Log::LEVEL_ERROR, :error],
|
32
|
+
fatal: [Fluent::Log::LEVEL_FATAL, :fatal])
|
33
|
+
def test_reflect_log_level(data)
|
34
|
+
level, expected = data
|
35
|
+
@fluent_log.level = level
|
36
|
+
console_logger = Fluent::Log::ConsoleAdapter.wrap(@fluent_log)
|
37
|
+
assert_equal(Console::Logger::LEVELS[expected],
|
38
|
+
console_logger.level)
|
39
|
+
end
|
40
|
+
|
41
|
+
data(debug: :debug,
|
42
|
+
info: :info,
|
43
|
+
warn: :warn,
|
44
|
+
error: :error,
|
45
|
+
fatal: :fatal)
|
46
|
+
def test_string_subject(level)
|
47
|
+
@console_logger.send(level, "subject")
|
48
|
+
assert_equal(["#{@timestamp_str} [#{level}]: 0.0s: subject\n"],
|
49
|
+
@logdev.logs)
|
50
|
+
end
|
51
|
+
|
52
|
+
data(debug: :debug,
|
53
|
+
info: :info,
|
54
|
+
warn: :warn,
|
55
|
+
error: :error,
|
56
|
+
fatal: :fatal)
|
57
|
+
def test_args(level)
|
58
|
+
@console_logger.send(level, "subject", 1, 2, 3)
|
59
|
+
assert_equal([
|
60
|
+
"#{@timestamp_str} [#{level}]: 0.0s: subject\n" +
|
61
|
+
" | 1\n" +
|
62
|
+
" | 2\n" +
|
63
|
+
" | 3\n"
|
64
|
+
],
|
65
|
+
@logdev.logs)
|
66
|
+
end
|
67
|
+
|
68
|
+
data(debug: :debug,
|
69
|
+
info: :info,
|
70
|
+
warn: :warn,
|
71
|
+
error: :error,
|
72
|
+
fatal: :fatal)
|
73
|
+
def test_options(level)
|
74
|
+
@console_logger.send(level, "subject", kwarg1: "opt1", kwarg2: "opt2")
|
75
|
+
assert_equal([
|
76
|
+
"#{@timestamp_str} [#{level}]: 0.0s: subject\n" +
|
77
|
+
" | {\"kwarg1\":\"opt1\",\"kwarg2\":\"opt2\"}\n"
|
78
|
+
],
|
79
|
+
@logdev.logs)
|
80
|
+
end
|
81
|
+
|
82
|
+
data(debug: :debug,
|
83
|
+
info: :info,
|
84
|
+
warn: :warn,
|
85
|
+
error: :error,
|
86
|
+
fatal: :fatal)
|
87
|
+
def test_block(level)
|
88
|
+
@console_logger.send(level, "subject") { "block message" }
|
89
|
+
assert_equal([
|
90
|
+
"#{@timestamp_str} [#{level}]: 0.0s: subject\n" +
|
91
|
+
" | block message\n"
|
92
|
+
],
|
93
|
+
@logdev.logs)
|
94
|
+
end
|
95
|
+
|
96
|
+
data(debug: :debug,
|
97
|
+
info: :info,
|
98
|
+
warn: :warn,
|
99
|
+
error: :error,
|
100
|
+
fatal: :fatal)
|
101
|
+
def test_multiple_entries(level)
|
102
|
+
@console_logger.send(level, "subject1")
|
103
|
+
@console_logger.send(level, "line2")
|
104
|
+
assert_equal([
|
105
|
+
"#{@timestamp_str} [#{level}]: 0.0s: subject1\n",
|
106
|
+
"#{@timestamp_str} [#{level}]: 0.0s: line2\n"
|
107
|
+
],
|
108
|
+
@logdev.logs)
|
109
|
+
end
|
110
|
+
end
|