fluentd 1.13.3 → 1.16.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/ISSUE_TEMPLATE/{bug_report.yaml → bug_report.yml} +2 -0
- data/.github/ISSUE_TEMPLATE/config.yml +2 -2
- data/.github/ISSUE_TEMPLATE/{feature_request.yaml → feature_request.yml} +1 -0
- data/.github/workflows/stale-actions.yml +11 -9
- data/.github/workflows/test.yml +32 -0
- data/CHANGELOG.md +490 -10
- data/CONTRIBUTING.md +2 -2
- data/MAINTAINERS.md +7 -5
- data/README.md +3 -23
- data/Rakefile +1 -1
- data/SECURITY.md +14 -0
- data/fluentd.gemspec +7 -8
- data/lib/fluent/command/cat.rb +13 -3
- data/lib/fluent/command/ctl.rb +6 -3
- data/lib/fluent/command/fluentd.rb +73 -65
- data/lib/fluent/command/plugin_config_formatter.rb +1 -1
- data/lib/fluent/compat/output.rb +9 -6
- data/lib/fluent/config/dsl.rb +1 -1
- data/lib/fluent/config/error.rb +12 -0
- data/lib/fluent/config/literal_parser.rb +2 -2
- data/lib/fluent/config/parser.rb +1 -1
- data/lib/fluent/config/v1_parser.rb +3 -3
- data/lib/fluent/config/yaml_parser/fluent_value.rb +47 -0
- data/lib/fluent/config/yaml_parser/loader.rb +108 -0
- data/lib/fluent/config/yaml_parser/parser.rb +166 -0
- data/lib/fluent/config/yaml_parser/section_builder.rb +107 -0
- data/lib/fluent/config/yaml_parser.rb +56 -0
- data/lib/fluent/config.rb +14 -1
- data/lib/fluent/counter/server.rb +1 -1
- data/lib/fluent/counter/validator.rb +3 -3
- data/lib/fluent/daemon.rb +2 -4
- data/lib/fluent/engine.rb +1 -1
- data/lib/fluent/env.rb +4 -0
- data/lib/fluent/error.rb +3 -0
- data/lib/fluent/event.rb +8 -4
- data/lib/fluent/event_router.rb +47 -2
- data/lib/fluent/file_wrapper.rb +137 -0
- data/lib/fluent/log/console_adapter.rb +66 -0
- data/lib/fluent/log.rb +44 -5
- data/lib/fluent/match.rb +1 -1
- data/lib/fluent/msgpack_factory.rb +6 -1
- data/lib/fluent/oj_options.rb +1 -2
- data/lib/fluent/plugin/bare_output.rb +49 -8
- data/lib/fluent/plugin/base.rb +26 -9
- data/lib/fluent/plugin/buf_file.rb +34 -5
- data/lib/fluent/plugin/buf_file_single.rb +32 -3
- data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
- data/lib/fluent/plugin/buffer.rb +216 -70
- data/lib/fluent/plugin/filter.rb +35 -1
- data/lib/fluent/plugin/filter_record_transformer.rb +1 -1
- data/lib/fluent/plugin/in_forward.rb +2 -2
- data/lib/fluent/plugin/in_http.rb +39 -10
- data/lib/fluent/plugin/in_monitor_agent.rb +4 -2
- data/lib/fluent/plugin/in_sample.rb +1 -1
- data/lib/fluent/plugin/in_syslog.rb +13 -1
- data/lib/fluent/plugin/in_tail/group_watch.rb +204 -0
- data/lib/fluent/plugin/in_tail/position_file.rb +33 -33
- data/lib/fluent/plugin/in_tail.rb +216 -84
- data/lib/fluent/plugin/in_tcp.rb +47 -2
- data/lib/fluent/plugin/input.rb +39 -1
- data/lib/fluent/plugin/metrics.rb +119 -0
- data/lib/fluent/plugin/metrics_local.rb +96 -0
- data/lib/fluent/plugin/multi_output.rb +43 -6
- data/lib/fluent/plugin/out_copy.rb +1 -1
- data/lib/fluent/plugin/out_exec_filter.rb +2 -2
- data/lib/fluent/plugin/out_file.rb +20 -2
- data/lib/fluent/plugin/out_forward/ack_handler.rb +19 -4
- data/lib/fluent/plugin/out_forward/socket_cache.rb +2 -0
- data/lib/fluent/plugin/out_forward.rb +17 -9
- data/lib/fluent/plugin/out_secondary_file.rb +39 -22
- data/lib/fluent/plugin/output.rb +167 -78
- data/lib/fluent/plugin/parser.rb +3 -4
- data/lib/fluent/plugin/parser_apache2.rb +1 -1
- data/lib/fluent/plugin/parser_json.rb +1 -1
- data/lib/fluent/plugin/parser_syslog.rb +1 -1
- data/lib/fluent/plugin/storage_local.rb +3 -5
- data/lib/fluent/plugin.rb +10 -1
- data/lib/fluent/plugin_helper/child_process.rb +3 -0
- data/lib/fluent/plugin_helper/event_emitter.rb +8 -1
- data/lib/fluent/plugin_helper/event_loop.rb +2 -2
- data/lib/fluent/plugin_helper/http_server/server.rb +2 -1
- data/lib/fluent/plugin_helper/metrics.rb +129 -0
- data/lib/fluent/plugin_helper/record_accessor.rb +1 -1
- data/lib/fluent/plugin_helper/retry_state.rb +14 -4
- data/lib/fluent/plugin_helper/server.rb +35 -6
- data/lib/fluent/plugin_helper/service_discovery.rb +2 -2
- data/lib/fluent/plugin_helper/socket.rb +13 -2
- data/lib/fluent/plugin_helper/thread.rb +3 -3
- data/lib/fluent/plugin_helper.rb +1 -0
- data/lib/fluent/plugin_id.rb +3 -2
- data/lib/fluent/registry.rb +2 -1
- data/lib/fluent/root_agent.rb +6 -0
- data/lib/fluent/rpc.rb +4 -3
- data/lib/fluent/supervisor.rb +283 -259
- data/lib/fluent/system_config.rb +13 -3
- data/lib/fluent/test/driver/base.rb +11 -5
- data/lib/fluent/test/driver/filter.rb +4 -0
- data/lib/fluent/test/startup_shutdown.rb +6 -8
- data/lib/fluent/time.rb +21 -20
- data/lib/fluent/version.rb +1 -1
- data/lib/fluent/win32api.rb +38 -0
- data/lib/fluent/winsvc.rb +5 -8
- data/templates/new_gem/test/helper.rb.erb +0 -1
- data/test/command/test_cat.rb +31 -2
- data/test/command/test_ctl.rb +1 -2
- data/test/command/test_fluentd.rb +209 -24
- data/test/command/test_plugin_config_formatter.rb +0 -1
- data/test/compat/test_parser.rb +6 -6
- data/test/config/test_system_config.rb +13 -11
- data/test/config/test_types.rb +1 -1
- data/test/log/test_console_adapter.rb +110 -0
- data/test/plugin/in_tail/test_io_handler.rb +26 -8
- data/test/plugin/in_tail/test_position_file.rb +48 -59
- data/test/plugin/out_forward/test_ack_handler.rb +39 -0
- data/test/plugin/out_forward/test_socket_cache.rb +26 -1
- data/test/plugin/test_bare_output.rb +14 -1
- data/test/plugin/test_base.rb +133 -1
- data/test/plugin/test_buf_file.rb +62 -23
- data/test/plugin/test_buf_file_single.rb +65 -0
- data/test/plugin/test_buffer.rb +267 -3
- data/test/plugin/test_buffer_chunk.rb +11 -0
- data/test/plugin/test_filter.rb +12 -1
- data/test/plugin/test_filter_parser.rb +1 -1
- data/test/plugin/test_filter_stdout.rb +2 -2
- data/test/plugin/test_in_forward.rb +9 -11
- data/test/plugin/test_in_http.rb +65 -3
- data/test/plugin/test_in_monitor_agent.rb +216 -11
- data/test/plugin/test_in_object_space.rb +9 -3
- data/test/plugin/test_in_syslog.rb +35 -0
- data/test/plugin/test_in_tail.rb +1393 -385
- data/test/plugin/test_in_tcp.rb +87 -2
- data/test/plugin/test_in_udp.rb +28 -0
- data/test/plugin/test_in_unix.rb +2 -2
- data/test/plugin/test_input.rb +12 -1
- data/test/plugin/test_metrics.rb +294 -0
- data/test/plugin/test_metrics_local.rb +96 -0
- data/test/plugin/test_multi_output.rb +25 -1
- data/test/plugin/test_out_exec.rb +6 -4
- data/test/plugin/test_out_exec_filter.rb +6 -2
- data/test/plugin/test_out_file.rb +34 -17
- data/test/plugin/test_out_forward.rb +78 -77
- data/test/plugin/test_out_http.rb +1 -0
- data/test/plugin/test_out_stdout.rb +2 -2
- data/test/plugin/test_output.rb +297 -12
- data/test/plugin/test_output_as_buffered.rb +44 -44
- data/test/plugin/test_output_as_buffered_compress.rb +32 -18
- data/test/plugin/test_output_as_buffered_retries.rb +54 -7
- data/test/plugin/test_output_as_buffered_secondary.rb +4 -4
- data/test/plugin/test_parser_regexp.rb +1 -6
- data/test/plugin/test_parser_syslog.rb +1 -1
- data/test/plugin_helper/test_cert_option.rb +1 -1
- data/test/plugin_helper/test_child_process.rb +38 -16
- data/test/plugin_helper/test_event_emitter.rb +29 -0
- data/test/plugin_helper/test_http_server_helper.rb +1 -1
- data/test/plugin_helper/test_metrics.rb +137 -0
- data/test/plugin_helper/test_retry_state.rb +602 -38
- data/test/plugin_helper/test_server.rb +78 -6
- data/test/plugin_helper/test_timer.rb +2 -2
- data/test/test_config.rb +191 -24
- data/test/test_event_router.rb +17 -0
- data/test/test_file_wrapper.rb +53 -0
- data/test/test_formatter.rb +24 -21
- data/test/test_log.rb +122 -40
- data/test/test_msgpack_factory.rb +32 -0
- data/test/test_plugin_classes.rb +102 -0
- data/test/test_root_agent.rb +30 -1
- data/test/test_supervisor.rb +477 -257
- data/test/test_time_parser.rb +22 -0
- metadata +55 -34
- data/.drone.yml +0 -35
- data/.github/workflows/issue-auto-closer.yml +0 -12
- data/.github/workflows/linux-test.yaml +0 -36
- data/.github/workflows/macos-test.yaml +0 -30
- data/.github/workflows/windows-test.yaml +0 -46
- data/.gitlab-ci.yml +0 -103
- data/lib/fluent/plugin/file_wrapper.rb +0 -187
- data/test/plugin/test_file_wrapper.rb +0 -126
- data/test/test_logger_initializer.rb +0 -46
|
@@ -5,20 +5,35 @@ require_relative '../helper'
|
|
|
5
5
|
|
|
6
6
|
require 'fileutils'
|
|
7
7
|
require 'timeout'
|
|
8
|
+
require 'securerandom'
|
|
9
|
+
require 'fluent/file_wrapper'
|
|
8
10
|
|
|
9
11
|
class TestFluentdCommand < ::Test::Unit::TestCase
|
|
10
|
-
TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/command/fluentd#{ENV['TEST_ENV_NUMBER']}")
|
|
11
12
|
SUPERVISOR_PID_PATTERN = /starting fluentd-[.0-9]+ pid=(\d+)/
|
|
12
13
|
WORKER_PID_PATTERN = /starting fluentd worker pid=(\d+) /
|
|
13
14
|
|
|
15
|
+
def tmp_dir
|
|
16
|
+
File.join(File.dirname(__FILE__), "..", "tmp", "command" "fluentd#{ENV['TEST_ENV_NUMBER']}", SecureRandom.hex(10))
|
|
17
|
+
end
|
|
18
|
+
|
|
14
19
|
setup do
|
|
15
|
-
|
|
16
|
-
FileUtils.mkdir_p(
|
|
20
|
+
@tmp_dir = tmp_dir
|
|
21
|
+
FileUtils.mkdir_p(@tmp_dir)
|
|
17
22
|
@supervisor_pid = nil
|
|
18
23
|
@worker_pids = []
|
|
19
24
|
ENV["TEST_RUBY_PATH"] = nil
|
|
20
25
|
end
|
|
21
26
|
|
|
27
|
+
teardown do
|
|
28
|
+
begin
|
|
29
|
+
FileUtils.rm_rf(@tmp_dir)
|
|
30
|
+
rescue Errno::EACCES
|
|
31
|
+
# It may occur on Windows because of delete pending state due to delayed GC.
|
|
32
|
+
# Ruby 3.2 or later doesn't ignore Errno::EACCES:
|
|
33
|
+
# https://github.com/ruby/ruby/commit/983115cf3c8f75b1afbe3274f02c1529e1ce3a81
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
22
37
|
def process_exist?(pid)
|
|
23
38
|
begin
|
|
24
39
|
r = Process.waitpid(pid, Process::WNOHANG)
|
|
@@ -30,17 +45,17 @@ class TestFluentdCommand < ::Test::Unit::TestCase
|
|
|
30
45
|
end
|
|
31
46
|
|
|
32
47
|
def create_conf_file(name, content, ext_enc = 'utf-8')
|
|
33
|
-
conf_path = File.join(
|
|
34
|
-
|
|
48
|
+
conf_path = File.join(@tmp_dir, name)
|
|
49
|
+
Fluent::FileWrapper.open(conf_path, "w:#{ext_enc}:utf-8") do |file|
|
|
35
50
|
file.write content
|
|
36
51
|
end
|
|
37
52
|
conf_path
|
|
38
53
|
end
|
|
39
54
|
|
|
40
55
|
def create_plugin_file(name, content)
|
|
41
|
-
file_path = File.join(
|
|
56
|
+
file_path = File.join(@tmp_dir, 'plugin', name)
|
|
42
57
|
FileUtils.mkdir_p(File.dirname(file_path))
|
|
43
|
-
|
|
58
|
+
Fluent::FileWrapper.open(file_path, 'w') do |file|
|
|
44
59
|
file.write content
|
|
45
60
|
end
|
|
46
61
|
file_path
|
|
@@ -56,8 +71,22 @@ class TestFluentdCommand < ::Test::Unit::TestCase
|
|
|
56
71
|
end
|
|
57
72
|
end
|
|
58
73
|
|
|
59
|
-
def
|
|
60
|
-
|
|
74
|
+
def process_kill(pid)
|
|
75
|
+
if Fluent.windows?
|
|
76
|
+
Process.kill(:KILL, pid) rescue nil
|
|
77
|
+
return
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
begin
|
|
81
|
+
Process.kill(:TERM, pid) rescue nil
|
|
82
|
+
Timeout.timeout(10){ sleep 0.1 while process_exist?(pid) }
|
|
83
|
+
rescue Timeout::Error
|
|
84
|
+
Process.kill(:KILL, pid) rescue nil
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def execute_command(cmdline, chdir=@tmp_dir, env = {})
|
|
89
|
+
null_stream = Fluent::FileWrapper.open(File::NULL, 'w')
|
|
61
90
|
gemfile_path = File.expand_path(File.dirname(__FILE__) + "../../../Gemfile")
|
|
62
91
|
|
|
63
92
|
env = { "BUNDLE_GEMFILE" => gemfile_path }.merge(env)
|
|
@@ -70,12 +99,12 @@ class TestFluentdCommand < ::Test::Unit::TestCase
|
|
|
70
99
|
yield pid, io
|
|
71
100
|
# p(here: "execute command", pid: pid, worker_pids: @worker_pids)
|
|
72
101
|
ensure
|
|
73
|
-
|
|
102
|
+
process_kill(pid)
|
|
74
103
|
if @supervisor_pid
|
|
75
|
-
|
|
104
|
+
process_kill(@supervisor_pid)
|
|
76
105
|
end
|
|
77
106
|
@worker_pids.each do |cpid|
|
|
78
|
-
|
|
107
|
+
process_kill(cpid)
|
|
79
108
|
end
|
|
80
109
|
# p(here: "execute command", pid: pid, exist: process_exist?(pid), worker_pids: @worker_pids, exists: @worker_pids.map{|i| process_exist?(i) })
|
|
81
110
|
Timeout.timeout(10){ sleep 0.1 while process_exist?(pid) }
|
|
@@ -97,16 +126,18 @@ class TestFluentdCommand < ::Test::Unit::TestCase
|
|
|
97
126
|
end
|
|
98
127
|
end
|
|
99
128
|
|
|
100
|
-
|
|
129
|
+
# ATTENTION: This stops taking logs when all `pattern_list` match or timeout,
|
|
130
|
+
# so `patterns_not_match` can test only logs up to that point.
|
|
131
|
+
def assert_log_matches(cmdline, *pattern_list, patterns_not_match: [], timeout: 20, env: {})
|
|
101
132
|
matched = false
|
|
102
133
|
matched_wrongly = false
|
|
103
134
|
assert_error_msg = ""
|
|
104
135
|
stdio_buf = ""
|
|
105
136
|
begin
|
|
106
|
-
execute_command(cmdline,
|
|
137
|
+
execute_command(cmdline, @tmp_dir, env) do |pid, stdout|
|
|
107
138
|
begin
|
|
108
139
|
waiting(timeout) do
|
|
109
|
-
while process_exist?(pid)
|
|
140
|
+
while process_exist?(pid)
|
|
110
141
|
readables, _, _ = IO.select([stdout], nil, nil, 1)
|
|
111
142
|
next unless readables
|
|
112
143
|
break if readables.first.eof?
|
|
@@ -118,6 +149,18 @@ class TestFluentdCommand < ::Test::Unit::TestCase
|
|
|
118
149
|
if pattern_list.all?{|ptn| lines.any?{|line| ptn.is_a?(Regexp) ? ptn.match(line) : line.include?(ptn) } }
|
|
119
150
|
matched = true
|
|
120
151
|
end
|
|
152
|
+
|
|
153
|
+
if Fluent.windows?
|
|
154
|
+
# https://github.com/fluent/fluentd/issues/4095
|
|
155
|
+
# On Windows, the initial process is different from the supervisor process,
|
|
156
|
+
# so we need to wait until `SUPERVISOR_PID_PATTERN` appears in the logs to get the pid.
|
|
157
|
+
# (Worker processes will be killed by the supervisor process, so we don't need it-)
|
|
158
|
+
break if matched && SUPERVISOR_PID_PATTERN =~ stdio_buf
|
|
159
|
+
else
|
|
160
|
+
# On Non-Windows, the initial process is the supervisor process,
|
|
161
|
+
# so we don't need to wait `SUPERVISOR_PID_PATTERN`.
|
|
162
|
+
break if matched
|
|
163
|
+
end
|
|
121
164
|
end
|
|
122
165
|
end
|
|
123
166
|
ensure
|
|
@@ -131,6 +174,10 @@ class TestFluentdCommand < ::Test::Unit::TestCase
|
|
|
131
174
|
end
|
|
132
175
|
rescue Timeout::Error
|
|
133
176
|
assert_error_msg = "execution timeout"
|
|
177
|
+
# https://github.com/fluent/fluentd/issues/4095
|
|
178
|
+
# On Windows, timeout without `@supervisor_pid` means that the test is invalid,
|
|
179
|
+
# since the supervisor process will survive without being killed correctly.
|
|
180
|
+
flunk("Invalid test: The pid of supervisor could not be taken, which is necessary on Windows.") if Fluent.windows? && @supervisor_pid.nil?
|
|
134
181
|
rescue => e
|
|
135
182
|
assert_error_msg = "unexpected error in launching fluentd: #{e.inspect}"
|
|
136
183
|
else
|
|
@@ -162,7 +209,7 @@ class TestFluentdCommand < ::Test::Unit::TestCase
|
|
|
162
209
|
assert matched && !matched_wrongly, assert_error_msg
|
|
163
210
|
end
|
|
164
211
|
|
|
165
|
-
def assert_fluentd_fails_to_start(cmdline, *pattern_list, timeout:
|
|
212
|
+
def assert_fluentd_fails_to_start(cmdline, *pattern_list, timeout: 20)
|
|
166
213
|
# empty_list.all?{ ... } is always true
|
|
167
214
|
matched = false
|
|
168
215
|
running = false
|
|
@@ -198,6 +245,10 @@ class TestFluentdCommand < ::Test::Unit::TestCase
|
|
|
198
245
|
end
|
|
199
246
|
rescue Timeout::Error
|
|
200
247
|
assert_error_msg = "execution timeout with command out:\n" + stdio_buf
|
|
248
|
+
# https://github.com/fluent/fluentd/issues/4095
|
|
249
|
+
# On Windows, timeout without `@supervisor_pid` means that the test is invalid,
|
|
250
|
+
# since the supervisor process will survive without being killed correctly.
|
|
251
|
+
flunk("Invalid test: The pid of supervisor could not be taken, which is necessary on Windows.") if Fluent.windows? && @supervisor_pid.nil?
|
|
201
252
|
rescue => e
|
|
202
253
|
assert_error_msg = "unexpected error in launching fluentd: #{e.inspect}\n" + stdio_buf
|
|
203
254
|
assert false, assert_error_msg
|
|
@@ -269,7 +320,7 @@ CONF
|
|
|
269
320
|
|
|
270
321
|
sub_test_case 'with system configuration about root directory' do
|
|
271
322
|
setup do
|
|
272
|
-
@root_path = File.join(
|
|
323
|
+
@root_path = File.join(@tmp_dir, "rootpath")
|
|
273
324
|
FileUtils.rm_rf(@root_path)
|
|
274
325
|
@conf = <<CONF
|
|
275
326
|
<system>
|
|
@@ -308,7 +359,7 @@ CONF
|
|
|
308
359
|
end
|
|
309
360
|
|
|
310
361
|
test 'fails to launch fluentd if specified root path is invalid path for directory' do
|
|
311
|
-
|
|
362
|
+
Fluent::FileWrapper.open(@root_path, 'w') do |_|
|
|
312
363
|
# create file and close it
|
|
313
364
|
end
|
|
314
365
|
conf_path = create_conf_file('existing_root_dir.conf', @conf)
|
|
@@ -508,7 +559,7 @@ CONF
|
|
|
508
559
|
|
|
509
560
|
assert_fluentd_fails_to_start(
|
|
510
561
|
create_cmdline(conf_path, "-p", File.dirname(plugin_path)),
|
|
511
|
-
"in_buggy.rb:5: syntax error, unexpected end-of-input
|
|
562
|
+
"in_buggy.rb:5: syntax error, unexpected end-of-input"
|
|
512
563
|
)
|
|
513
564
|
end
|
|
514
565
|
end
|
|
@@ -554,7 +605,7 @@ CONF
|
|
|
554
605
|
|
|
555
606
|
sub_test_case 'configured to run 2 workers' do
|
|
556
607
|
setup do
|
|
557
|
-
@root_path = File.join(
|
|
608
|
+
@root_path = File.join(@tmp_dir, "rootpath")
|
|
558
609
|
FileUtils.rm_rf(@root_path)
|
|
559
610
|
FileUtils.mkdir_p(@root_path)
|
|
560
611
|
end
|
|
@@ -589,6 +640,39 @@ CONF
|
|
|
589
640
|
)
|
|
590
641
|
end
|
|
591
642
|
|
|
643
|
+
sub_test_case "YAML config format" do
|
|
644
|
+
test 'success to start the number of workers specified in configuration' do
|
|
645
|
+
conf = <<'CONF'
|
|
646
|
+
system:
|
|
647
|
+
workers: 2
|
|
648
|
+
root_dir: "#{@root_path}"
|
|
649
|
+
config:
|
|
650
|
+
- source:
|
|
651
|
+
$type: dummy
|
|
652
|
+
$id: !fluent/s "dummy.#{worker_id}" # check worker_id works or not with actual command
|
|
653
|
+
$label: '@dummydata'
|
|
654
|
+
tag: dummy
|
|
655
|
+
dummy: !fluent/json {"message": !fluent/s "yay from #{hostname}!"}
|
|
656
|
+
|
|
657
|
+
- label:
|
|
658
|
+
$name: '@dummydata'
|
|
659
|
+
config:
|
|
660
|
+
- match:
|
|
661
|
+
$tag: dummy
|
|
662
|
+
$type: "null"
|
|
663
|
+
$id: blackhole
|
|
664
|
+
CONF
|
|
665
|
+
conf_path = create_conf_file('workers1.yaml', conf)
|
|
666
|
+
assert Dir.exist?(@root_path)
|
|
667
|
+
|
|
668
|
+
assert_log_matches(
|
|
669
|
+
create_cmdline(conf_path),
|
|
670
|
+
"#0 fluentd worker is now running worker=0",
|
|
671
|
+
"#1 fluentd worker is now running worker=1"
|
|
672
|
+
)
|
|
673
|
+
end
|
|
674
|
+
end
|
|
675
|
+
|
|
592
676
|
test 'success to start the number of workers specified by command line option' do
|
|
593
677
|
conf = <<CONF
|
|
594
678
|
<system>
|
|
@@ -857,7 +941,7 @@ CONF
|
|
|
857
941
|
'-external-encoding' => '--external-encoding=utf-8',
|
|
858
942
|
'-internal-encoding' => '--internal-encoding=utf-8',
|
|
859
943
|
)
|
|
860
|
-
test "-E option is set to RUBYOPT" do |
|
|
944
|
+
test "-E option is set to RUBYOPT" do |base_opt|
|
|
861
945
|
conf = <<CONF
|
|
862
946
|
<source>
|
|
863
947
|
@type dummy
|
|
@@ -868,6 +952,7 @@ CONF
|
|
|
868
952
|
</match>
|
|
869
953
|
CONF
|
|
870
954
|
conf_path = create_conf_file('rubyopt_test.conf', conf)
|
|
955
|
+
opt = base_opt.dup
|
|
871
956
|
opt << " #{ENV['RUBYOPT']}" if ENV['RUBYOPT']
|
|
872
957
|
assert_log_matches(
|
|
873
958
|
create_cmdline(conf_path),
|
|
@@ -907,9 +992,14 @@ CONF
|
|
|
907
992
|
</match>
|
|
908
993
|
CONF
|
|
909
994
|
conf_path = create_conf_file('rubyopt_invalid_test.conf', conf)
|
|
995
|
+
if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('3.3.0')
|
|
996
|
+
expected_phrase = 'ruby: invalid switch in RUBYOPT'
|
|
997
|
+
else
|
|
998
|
+
expected_phrase = 'Invalid option is passed to RUBYOPT'
|
|
999
|
+
end
|
|
910
1000
|
assert_log_matches(
|
|
911
1001
|
create_cmdline(conf_path),
|
|
912
|
-
|
|
1002
|
+
expected_phrase,
|
|
913
1003
|
env: { 'RUBYOPT' => 'a' },
|
|
914
1004
|
)
|
|
915
1005
|
end
|
|
@@ -928,10 +1018,10 @@ CONF
|
|
|
928
1018
|
</match>
|
|
929
1019
|
CONF
|
|
930
1020
|
ruby_path = ServerEngine.ruby_bin_path
|
|
931
|
-
tmp_ruby_path = File.join(
|
|
1021
|
+
tmp_ruby_path = File.join(@tmp_dir, "ruby with spaces")
|
|
932
1022
|
if Fluent.windows?
|
|
933
1023
|
tmp_ruby_path << ".bat"
|
|
934
|
-
|
|
1024
|
+
Fluent::FileWrapper.open(tmp_ruby_path, "w") do |file|
|
|
935
1025
|
file.write "#{ruby_path} %*"
|
|
936
1026
|
end
|
|
937
1027
|
else
|
|
@@ -1103,4 +1193,99 @@ CONF
|
|
|
1103
1193
|
"shared socket for multiple workers is disabled",)
|
|
1104
1194
|
end
|
|
1105
1195
|
end
|
|
1196
|
+
|
|
1197
|
+
# TODO: `patterns_not_match` can test only logs up to `pattern_list`,
|
|
1198
|
+
# so we need to fix some meaningless `patterns_not_match` conditions.
|
|
1199
|
+
sub_test_case 'log_level by command line option' do
|
|
1200
|
+
test 'info' do
|
|
1201
|
+
conf = ""
|
|
1202
|
+
conf_path = create_conf_file('empty.conf', conf)
|
|
1203
|
+
assert File.exist?(conf_path)
|
|
1204
|
+
assert_log_matches(create_cmdline(conf_path),
|
|
1205
|
+
"[info]",
|
|
1206
|
+
patterns_not_match: ["[debug]"])
|
|
1207
|
+
end
|
|
1208
|
+
|
|
1209
|
+
test 'debug' do
|
|
1210
|
+
conf = ""
|
|
1211
|
+
conf_path = create_conf_file('empty.conf', conf)
|
|
1212
|
+
assert File.exist?(conf_path)
|
|
1213
|
+
assert_log_matches(create_cmdline(conf_path, "-v"),
|
|
1214
|
+
"[debug]",
|
|
1215
|
+
patterns_not_match: ["[trace]"])
|
|
1216
|
+
end
|
|
1217
|
+
|
|
1218
|
+
data("Trace" => "-vv")
|
|
1219
|
+
data("Invalid low level should be treated as Trace level": "-vvv")
|
|
1220
|
+
test 'trace' do |option|
|
|
1221
|
+
conf = <<CONF
|
|
1222
|
+
<source>
|
|
1223
|
+
@type sample
|
|
1224
|
+
tag test
|
|
1225
|
+
</source>
|
|
1226
|
+
CONF
|
|
1227
|
+
conf_path = create_conf_file('sample.conf', conf)
|
|
1228
|
+
assert File.exist?(conf_path)
|
|
1229
|
+
assert_log_matches(create_cmdline(conf_path, option),
|
|
1230
|
+
"[trace]",)
|
|
1231
|
+
end
|
|
1232
|
+
|
|
1233
|
+
test 'warn' do
|
|
1234
|
+
omit "Can't run on Windows since there is no way to take pid of the supervisor." if Fluent.windows?
|
|
1235
|
+
conf = <<CONF
|
|
1236
|
+
<source>
|
|
1237
|
+
@type sample
|
|
1238
|
+
tag test
|
|
1239
|
+
</source>
|
|
1240
|
+
CONF
|
|
1241
|
+
conf_path = create_conf_file('sample.conf', conf)
|
|
1242
|
+
assert File.exist?(conf_path)
|
|
1243
|
+
assert_log_matches(create_cmdline(conf_path, "-q"),
|
|
1244
|
+
"[warn]",
|
|
1245
|
+
patterns_not_match: ["[info]"])
|
|
1246
|
+
end
|
|
1247
|
+
|
|
1248
|
+
data("Error" => "-qq")
|
|
1249
|
+
data("Fatal should be treated as Error level" => "-qqq")
|
|
1250
|
+
data("Invalid high level should be treated as Error level": "-qqqq")
|
|
1251
|
+
test 'error' do |option|
|
|
1252
|
+
# This test can run on Windows correctly,
|
|
1253
|
+
# since the process will stop automatically with an error.
|
|
1254
|
+
conf = <<CONF
|
|
1255
|
+
<source>
|
|
1256
|
+
@type plugin_not_found
|
|
1257
|
+
tag test
|
|
1258
|
+
</source>
|
|
1259
|
+
CONF
|
|
1260
|
+
conf_path = create_conf_file('plugin_not_found.conf', conf)
|
|
1261
|
+
assert File.exist?(conf_path)
|
|
1262
|
+
assert_log_matches(create_cmdline(conf_path, option),
|
|
1263
|
+
"[error]",
|
|
1264
|
+
patterns_not_match: ["[warn]"])
|
|
1265
|
+
end
|
|
1266
|
+
|
|
1267
|
+
test 'system config one should not be overwritten when cmd line one is not specified' do
|
|
1268
|
+
conf = <<CONF
|
|
1269
|
+
<system>
|
|
1270
|
+
log_level debug
|
|
1271
|
+
</system>
|
|
1272
|
+
CONF
|
|
1273
|
+
conf_path = create_conf_file('debug.conf', conf)
|
|
1274
|
+
assert File.exist?(conf_path)
|
|
1275
|
+
assert_log_matches(create_cmdline(conf_path),
|
|
1276
|
+
"[debug]")
|
|
1277
|
+
end
|
|
1278
|
+
end
|
|
1279
|
+
|
|
1280
|
+
sub_test_case "inline_config" do
|
|
1281
|
+
test "can change log_level by --inline-config" do
|
|
1282
|
+
# Since we can't define multiple `<system>` directives, this use-case is not recommended.
|
|
1283
|
+
# This is just for this test.
|
|
1284
|
+
inline_conf = '<system>\nlog_level debug\n</system>'
|
|
1285
|
+
conf_path = create_conf_file('test.conf', "")
|
|
1286
|
+
assert File.exist?(conf_path)
|
|
1287
|
+
assert_log_matches(create_cmdline(conf_path, "--inline-config", inline_conf),
|
|
1288
|
+
"[debug]")
|
|
1289
|
+
end
|
|
1290
|
+
end
|
|
1106
1291
|
end
|
|
@@ -188,7 +188,6 @@ slow_flush_log_threshold: float: (20.0)
|
|
|
188
188
|
retry_exponential_backoff_base: float: (2)
|
|
189
189
|
retry_max_interval: time: (nil)
|
|
190
190
|
retry_randomize: bool: (true)
|
|
191
|
-
disable_chunk_backup: bool: (false)
|
|
192
191
|
<secondary>: optional, single
|
|
193
192
|
@type: string: (nil)
|
|
194
193
|
<buffer>: optional, single
|
data/test/compat/test_parser.rb
CHANGED
|
@@ -22,7 +22,7 @@ class TextParserTest < ::Test::Unit::TestCase
|
|
|
22
22
|
Fluent::TextParser.register_template('multi_event_test', Proc.new { MultiEventTestParser.new })
|
|
23
23
|
|
|
24
24
|
def test_lookup_unknown_format
|
|
25
|
-
assert_raise Fluent::
|
|
25
|
+
assert_raise Fluent::NotFoundPluginError do
|
|
26
26
|
Fluent::Plugin.new_parser('unknown')
|
|
27
27
|
end
|
|
28
28
|
end
|
|
@@ -38,14 +38,14 @@ class TextParserTest < ::Test::Unit::TestCase
|
|
|
38
38
|
|
|
39
39
|
def test_parse_with_return
|
|
40
40
|
parser = Fluent::TextParser.new
|
|
41
|
-
parser.configure('format' => 'none')
|
|
41
|
+
parser.configure(config_element('test', '', 'format' => 'none'))
|
|
42
42
|
_time, record = parser.parse('log message!')
|
|
43
43
|
assert_equal({'message' => 'log message!'}, record)
|
|
44
44
|
end
|
|
45
45
|
|
|
46
46
|
def test_parse_with_block
|
|
47
47
|
parser = Fluent::TextParser.new
|
|
48
|
-
parser.configure('format' => 'none')
|
|
48
|
+
parser.configure(config_element('test', '', 'format' => 'none'))
|
|
49
49
|
parser.parse('log message!') { |time, record|
|
|
50
50
|
assert_equal({'message' => 'log message!'}, record)
|
|
51
51
|
}
|
|
@@ -53,7 +53,7 @@ class TextParserTest < ::Test::Unit::TestCase
|
|
|
53
53
|
|
|
54
54
|
def test_multi_event_parser
|
|
55
55
|
parser = Fluent::TextParser.new
|
|
56
|
-
parser.configure('format' => 'multi_event_test')
|
|
56
|
+
parser.configure(config_element('test', '', 'format' => 'multi_event_test'))
|
|
57
57
|
i = 0
|
|
58
58
|
parser.parse('log message!') { |time, record|
|
|
59
59
|
assert_equal('log message!', record['message'])
|
|
@@ -67,7 +67,7 @@ class TextParserTest < ::Test::Unit::TestCase
|
|
|
67
67
|
assert_nil p1.estimate_current_event
|
|
68
68
|
assert_nil p1.parser
|
|
69
69
|
|
|
70
|
-
p1.configure('format' => 'none')
|
|
70
|
+
p1.configure(config_element('test', '', 'format' => 'none'))
|
|
71
71
|
assert_equal true, p1.parser.estimate_current_event
|
|
72
72
|
|
|
73
73
|
p2 = Fluent::TextParser.new
|
|
@@ -76,7 +76,7 @@ class TextParserTest < ::Test::Unit::TestCase
|
|
|
76
76
|
|
|
77
77
|
p2.estimate_current_event = false
|
|
78
78
|
|
|
79
|
-
p2.configure('format' => 'none')
|
|
79
|
+
p2.configure(config_element('test', '', 'format' => 'none'))
|
|
80
80
|
assert_equal false, p2.parser.estimate_current_event
|
|
81
81
|
end
|
|
82
82
|
|
|
@@ -5,13 +5,6 @@ require 'fluent/config/section'
|
|
|
5
5
|
require 'fluent/system_config'
|
|
6
6
|
|
|
7
7
|
module Fluent::Config
|
|
8
|
-
class FakeLoggerInitializer
|
|
9
|
-
attr_accessor :level
|
|
10
|
-
def initialize
|
|
11
|
-
@level = nil
|
|
12
|
-
end
|
|
13
|
-
end
|
|
14
|
-
|
|
15
8
|
class FakeSupervisor
|
|
16
9
|
attr_writer :log_level
|
|
17
10
|
|
|
@@ -19,8 +12,8 @@ module Fluent::Config
|
|
|
19
12
|
@system_config = nil
|
|
20
13
|
@cl_opt = {
|
|
21
14
|
wokers: nil,
|
|
15
|
+
restart_worker_interval: nil,
|
|
22
16
|
root_dir: nil,
|
|
23
|
-
log: FakeLoggerInitializer.new,
|
|
24
17
|
log_level: Fluent::Log::LEVEL_INFO,
|
|
25
18
|
suppress_interval: nil,
|
|
26
19
|
suppress_config_dump: nil,
|
|
@@ -28,6 +21,8 @@ module Fluent::Config
|
|
|
28
21
|
log_event_label: nil,
|
|
29
22
|
log_event_verbose: nil,
|
|
30
23
|
without_source: nil,
|
|
24
|
+
enable_input_metrics: nil,
|
|
25
|
+
enable_size_metrics: nil,
|
|
31
26
|
emit_error_log_interval: nil,
|
|
32
27
|
file_permission: nil,
|
|
33
28
|
dir_permission: nil,
|
|
@@ -70,6 +65,7 @@ module Fluent::Config
|
|
|
70
65
|
sc = Fluent::SystemConfig.new(conf)
|
|
71
66
|
sc.overwrite_variables(**s.for_system_config)
|
|
72
67
|
assert_equal(1, sc.workers)
|
|
68
|
+
assert_equal(0, sc.restart_worker_interval)
|
|
73
69
|
assert_nil(sc.root_dir)
|
|
74
70
|
assert_equal(Fluent::Log::LEVEL_INFO, sc.log_level)
|
|
75
71
|
assert_nil(sc.suppress_repeated_stacktrace)
|
|
@@ -77,13 +73,17 @@ module Fluent::Config
|
|
|
77
73
|
assert_nil(sc.emit_error_log_interval)
|
|
78
74
|
assert_nil(sc.suppress_config_dump)
|
|
79
75
|
assert_nil(sc.without_source)
|
|
76
|
+
assert_nil(sc.enable_input_metrics)
|
|
77
|
+
assert_nil(sc.enable_size_metrics)
|
|
80
78
|
assert_nil(sc.enable_msgpack_time_support)
|
|
79
|
+
assert(!sc.enable_jit)
|
|
81
80
|
assert_equal(:text, sc.log.format)
|
|
82
81
|
assert_equal('%Y-%m-%d %H:%M:%S %z', sc.log.time_format)
|
|
83
82
|
end
|
|
84
83
|
|
|
85
84
|
data(
|
|
86
85
|
'workers' => ['workers', 3],
|
|
86
|
+
'restart_worker_interval' => ['restart_worker_interval', 60],
|
|
87
87
|
'root_dir' => ['root_dir', File.join(TMP_DIR, 'root')],
|
|
88
88
|
'log_level' => ['log_level', 'error'],
|
|
89
89
|
'suppress_repeated_stacktrace' => ['suppress_repeated_stacktrace', true],
|
|
@@ -93,6 +93,9 @@ module Fluent::Config
|
|
|
93
93
|
'without_source' => ['without_source', true],
|
|
94
94
|
'strict_config_value' => ['strict_config_value', true],
|
|
95
95
|
'enable_msgpack_time_support' => ['enable_msgpack_time_support', true],
|
|
96
|
+
'enable_input_metrics' => ['enable_input_metrics', true],
|
|
97
|
+
'enable_size_metrics' => ['enable_size_metrics', true],
|
|
98
|
+
'enable_jit' => ['enable_jit', true],
|
|
96
99
|
)
|
|
97
100
|
test "accepts parameters" do |(k, v)|
|
|
98
101
|
conf = parse_text(<<-EOS)
|
|
@@ -148,7 +151,7 @@ module Fluent::Config
|
|
|
148
151
|
data('daily' => "daily",
|
|
149
152
|
'weekly' => 'weekly',
|
|
150
153
|
'monthly' => 'monthly')
|
|
151
|
-
test "
|
|
154
|
+
test "strings for rotate_age" do |age|
|
|
152
155
|
conf = parse_text(<<-EOS)
|
|
153
156
|
<system>
|
|
154
157
|
<log>
|
|
@@ -157,7 +160,7 @@ module Fluent::Config
|
|
|
157
160
|
</system>
|
|
158
161
|
EOS
|
|
159
162
|
sc = Fluent::SystemConfig.new(conf)
|
|
160
|
-
assert_equal(age
|
|
163
|
+
assert_equal(age, sc.log.rotate_age)
|
|
161
164
|
end
|
|
162
165
|
|
|
163
166
|
test "numeric number for rotate age" do
|
|
@@ -168,7 +171,6 @@ module Fluent::Config
|
|
|
168
171
|
</log>
|
|
169
172
|
</system>
|
|
170
173
|
EOS
|
|
171
|
-
s = FakeSupervisor.new
|
|
172
174
|
sc = Fluent::SystemConfig.new(conf)
|
|
173
175
|
assert_equal(3, sc.log.rotate_age)
|
|
174
176
|
end
|
data/test/config/test_types.rb
CHANGED
|
@@ -190,7 +190,7 @@ class TestConfigTypes < ::Test::Unit::TestCase
|
|
|
190
190
|
data("val" => [:val, 'val'],
|
|
191
191
|
"v" => [:v, 'v'],
|
|
192
192
|
"value" => [:value, 'value'])
|
|
193
|
-
test 'enum' do |(expected, val
|
|
193
|
+
test 'enum' do |(expected, val)|
|
|
194
194
|
assert_equal expected, Config::ENUM_TYPE.call(val, {list: [:val, :value, :v]})
|
|
195
195
|
end
|
|
196
196
|
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
require_relative '../helper'
|
|
2
|
+
|
|
3
|
+
require 'fluent/log'
|
|
4
|
+
require 'fluent/log/console_adapter'
|
|
5
|
+
|
|
6
|
+
class ConsoleAdapterTest < Test::Unit::TestCase
|
|
7
|
+
def setup
|
|
8
|
+
@timestamp = Time.parse("2023-01-01 15:32:41 +0000")
|
|
9
|
+
@timestamp_str = @timestamp.strftime("%Y-%m-%d %H:%M:%S %z")
|
|
10
|
+
Timecop.freeze(@timestamp)
|
|
11
|
+
|
|
12
|
+
@logdev = Fluent::Test::DummyLogDevice.new
|
|
13
|
+
@logger = ServerEngine::DaemonLogger.new(@logdev)
|
|
14
|
+
@fluent_log = Fluent::Log.new(@logger)
|
|
15
|
+
@console_logger = Fluent::Log::ConsoleAdapter.wrap(@fluent_log)
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def teardown
|
|
19
|
+
Timecop.return
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def test_expected_log_levels
|
|
23
|
+
assert_equal({debug: 0, info: 1, warn: 2, error: 3, fatal: 4},
|
|
24
|
+
Console::Logger::LEVELS)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
data(trace: [Fluent::Log::LEVEL_TRACE, :debug],
|
|
28
|
+
debug: [Fluent::Log::LEVEL_DEBUG, :debug],
|
|
29
|
+
info: [Fluent::Log::LEVEL_INFO, :info],
|
|
30
|
+
warn: [Fluent::Log::LEVEL_WARN, :warn],
|
|
31
|
+
error: [Fluent::Log::LEVEL_ERROR, :error],
|
|
32
|
+
fatal: [Fluent::Log::LEVEL_FATAL, :fatal])
|
|
33
|
+
def test_reflect_log_level(data)
|
|
34
|
+
level, expected = data
|
|
35
|
+
@fluent_log.level = level
|
|
36
|
+
console_logger = Fluent::Log::ConsoleAdapter.wrap(@fluent_log)
|
|
37
|
+
assert_equal(Console::Logger::LEVELS[expected],
|
|
38
|
+
console_logger.level)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
data(debug: :debug,
|
|
42
|
+
info: :info,
|
|
43
|
+
warn: :warn,
|
|
44
|
+
error: :error,
|
|
45
|
+
fatal: :fatal)
|
|
46
|
+
def test_string_subject(level)
|
|
47
|
+
@console_logger.send(level, "subject")
|
|
48
|
+
assert_equal(["#{@timestamp_str} [#{level}]: 0.0s: subject\n"],
|
|
49
|
+
@logdev.logs)
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
data(debug: :debug,
|
|
53
|
+
info: :info,
|
|
54
|
+
warn: :warn,
|
|
55
|
+
error: :error,
|
|
56
|
+
fatal: :fatal)
|
|
57
|
+
def test_args(level)
|
|
58
|
+
@console_logger.send(level, "subject", 1, 2, 3)
|
|
59
|
+
assert_equal([
|
|
60
|
+
"#{@timestamp_str} [#{level}]: 0.0s: subject\n" +
|
|
61
|
+
" | 1\n" +
|
|
62
|
+
" | 2\n" +
|
|
63
|
+
" | 3\n"
|
|
64
|
+
],
|
|
65
|
+
@logdev.logs)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
data(debug: :debug,
|
|
69
|
+
info: :info,
|
|
70
|
+
warn: :warn,
|
|
71
|
+
error: :error,
|
|
72
|
+
fatal: :fatal)
|
|
73
|
+
def test_options(level)
|
|
74
|
+
@console_logger.send(level, "subject", kwarg1: "opt1", kwarg2: "opt2")
|
|
75
|
+
assert_equal([
|
|
76
|
+
"#{@timestamp_str} [#{level}]: 0.0s: subject\n" +
|
|
77
|
+
" | {\"kwarg1\":\"opt1\",\"kwarg2\":\"opt2\"}\n"
|
|
78
|
+
],
|
|
79
|
+
@logdev.logs)
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
data(debug: :debug,
|
|
83
|
+
info: :info,
|
|
84
|
+
warn: :warn,
|
|
85
|
+
error: :error,
|
|
86
|
+
fatal: :fatal)
|
|
87
|
+
def test_block(level)
|
|
88
|
+
@console_logger.send(level, "subject") { "block message" }
|
|
89
|
+
assert_equal([
|
|
90
|
+
"#{@timestamp_str} [#{level}]: 0.0s: subject\n" +
|
|
91
|
+
" | block message\n"
|
|
92
|
+
],
|
|
93
|
+
@logdev.logs)
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
data(debug: :debug,
|
|
97
|
+
info: :info,
|
|
98
|
+
warn: :warn,
|
|
99
|
+
error: :error,
|
|
100
|
+
fatal: :fatal)
|
|
101
|
+
def test_multiple_entries(level)
|
|
102
|
+
@console_logger.send(level, "subject1")
|
|
103
|
+
@console_logger.send(level, "line2")
|
|
104
|
+
assert_equal([
|
|
105
|
+
"#{@timestamp_str} [#{level}]: 0.0s: subject1\n",
|
|
106
|
+
"#{@timestamp_str} [#{level}]: 0.0s: line2\n"
|
|
107
|
+
],
|
|
108
|
+
@logdev.logs)
|
|
109
|
+
end
|
|
110
|
+
end
|