fluentd 1.9.3 → 1.11.1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.github/workflows/issue-auto-closer.yml +12 -0
- data/CHANGELOG.md +182 -0
- data/CONTRIBUTING.md +1 -1
- data/README.md +4 -0
- data/docs/SECURITY_AUDIT.pdf +0 -0
- data/lib/fluent/command/debug.rb +1 -0
- data/lib/fluent/command/fluentd.rb +25 -1
- data/lib/fluent/config.rb +1 -0
- data/lib/fluent/daemonizer.rb +88 -0
- data/lib/fluent/log.rb +45 -6
- data/lib/fluent/match.rb +1 -1
- data/lib/fluent/msgpack_factory.rb +13 -6
- data/lib/fluent/plugin/buffer.rb +2 -2
- data/lib/fluent/plugin/in_dummy.rb +3 -3
- data/lib/fluent/plugin/in_forward.rb +2 -2
- data/lib/fluent/plugin/in_gc_stat.rb +16 -0
- data/lib/fluent/plugin/in_http.rb +146 -75
- data/lib/fluent/plugin/in_monitor_agent.rb +1 -1
- data/lib/fluent/plugin/in_syslog.rb +4 -4
- data/lib/fluent/plugin/in_tail.rb +40 -31
- data/lib/fluent/plugin/in_tail/position_file.rb +23 -6
- data/lib/fluent/plugin/in_unix.rb +77 -77
- data/lib/fluent/plugin/out_copy.rb +1 -1
- data/lib/fluent/plugin/out_file.rb +1 -1
- data/lib/fluent/plugin/out_forward.rb +25 -22
- data/lib/fluent/plugin/out_forward/handshake_protocol.rb +4 -0
- data/lib/fluent/plugin/out_forward/load_balancer.rb +1 -1
- data/lib/fluent/plugin/out_http.rb +15 -2
- data/lib/fluent/plugin/parser_multiline.rb +1 -1
- data/lib/fluent/plugin/parser_syslog.rb +303 -62
- data/lib/fluent/plugin/sd_file.rb +1 -0
- data/lib/fluent/plugin/sd_srv.rb +135 -0
- data/lib/fluent/plugin_helper/cert_option.rb +15 -2
- data/lib/fluent/plugin_helper/child_process.rb +3 -2
- data/lib/fluent/plugin_helper/record_accessor.rb +14 -0
- data/lib/fluent/plugin_helper/server.rb +3 -1
- data/lib/fluent/plugin_helper/service_discovery.rb +7 -0
- data/lib/fluent/plugin_helper/service_discovery/manager.rb +8 -0
- data/lib/fluent/plugin_helper/socket.rb +20 -2
- data/lib/fluent/plugin_helper/socket_option.rb +21 -3
- data/lib/fluent/supervisor.rb +21 -9
- data/lib/fluent/system_config.rb +2 -1
- data/lib/fluent/test/filter_test.rb +2 -2
- data/lib/fluent/test/output_test.rb +3 -3
- data/lib/fluent/version.rb +1 -1
- data/test/command/test_fluentd.rb +71 -12
- data/test/config/test_system_config.rb +2 -0
- data/test/helper.rb +2 -2
- data/test/plugin/in_tail/test_fifo.rb +121 -0
- data/test/plugin/in_tail/test_io_handler.rb +132 -0
- data/test/plugin/in_tail/test_position_file.rb +25 -1
- data/test/plugin/out_forward/test_handshake_protocol.rb +10 -1
- data/test/plugin/out_forward/test_load_balancer.rb +46 -0
- data/test/plugin/test_buf_file.rb +3 -1
- data/test/plugin/test_buffer.rb +20 -0
- data/test/plugin/test_compressable.rb +7 -4
- data/test/plugin/test_in_dummy.rb +12 -14
- data/test/plugin/test_in_forward.rb +2 -2
- data/test/plugin/test_in_gc_stat.rb +24 -1
- data/test/plugin/test_in_http.rb +57 -0
- data/test/plugin/test_in_syslog.rb +16 -1
- data/test/plugin/test_in_tail.rb +43 -20
- data/test/plugin/test_in_unix.rb +128 -73
- data/test/plugin/test_out_forward.rb +39 -3
- data/test/plugin/test_out_http.rb +38 -0
- data/test/plugin/test_out_null.rb +1 -1
- data/test/plugin/test_output_as_buffered_retries.rb +12 -4
- data/test/plugin/test_output_as_buffered_secondary.rb +9 -1
- data/test/plugin/test_parser_syslog.rb +106 -46
- data/test/plugin/test_sd_file.rb +17 -0
- data/test/plugin/test_sd_srv.rb +230 -0
- data/test/plugin_helper/data/cert/cert-with-CRLF.pem +19 -0
- data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +20 -0
- data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +27 -0
- data/test/plugin_helper/data/cert/cert_chains/cert.pem +40 -0
- data/test/plugin_helper/data/cert/generate_cert.rb +38 -0
- data/test/plugin_helper/http_server/test_app.rb +1 -1
- data/test/plugin_helper/http_server/test_route.rb +1 -1
- data/test/plugin_helper/test_cert_option.rb +2 -0
- data/test/plugin_helper/test_child_process.rb +20 -3
- data/test/plugin_helper/test_http_server_helper.rb +2 -2
- data/test/plugin_helper/test_record_accessor.rb +41 -0
- data/test/plugin_helper/test_server.rb +1 -1
- data/test/plugin_helper/test_service_discovery.rb +37 -4
- data/test/plugin_helper/test_socket.rb +131 -0
- data/test/test_daemonizer.rb +91 -0
- data/test/test_log.rb +44 -0
- data/test/test_msgpack_factory.rb +18 -0
- metadata +28 -2
@@ -0,0 +1,91 @@
|
|
1
|
+
require_relative 'helper'
|
2
|
+
require 'fluent/daemonizer'
|
3
|
+
|
4
|
+
class DaemonizerTest < ::Test::Unit::TestCase
|
5
|
+
TMP_DIR = File.join(File.dirname(__FILE__), 'tmp', 'daemonizer')
|
6
|
+
|
7
|
+
setup do
|
8
|
+
FileUtils.mkdir_p(TMP_DIR)
|
9
|
+
end
|
10
|
+
|
11
|
+
teardown do
|
12
|
+
FileUtils.rm_rf(TMP_DIR) rescue nil
|
13
|
+
end
|
14
|
+
|
15
|
+
test 'makes pid file' do
|
16
|
+
pid_path = File.join(TMP_DIR, 'file.pid')
|
17
|
+
|
18
|
+
mock(Process).daemon(anything, anything).once
|
19
|
+
r = Fluent::Daemonizer.daemonize(pid_path) { 'ret' }
|
20
|
+
assert_equal 'ret', r
|
21
|
+
assert File.exist?(pid_path)
|
22
|
+
assert Process.pid.to_s, File.read(pid_path).to_s
|
23
|
+
end
|
24
|
+
|
25
|
+
test 'in platforms which do not support fork' do
|
26
|
+
pid_path = File.join(TMP_DIR, 'file.pid')
|
27
|
+
|
28
|
+
mock(Process).daemon(anything, anything) { raise NotImplementedError }
|
29
|
+
args = ['-c', 'test.conf']
|
30
|
+
mock(Process).spawn(anything, *args) { Process.pid }
|
31
|
+
|
32
|
+
Fluent::Daemonizer.daemonize(pid_path, args) { 'ret' }
|
33
|
+
assert File.exist?(pid_path)
|
34
|
+
assert Process.pid.to_s, File.read(pid_path).to_s
|
35
|
+
end
|
36
|
+
|
37
|
+
sub_test_case 'when pid file already exists' do
|
38
|
+
test 'raise an error when process is running' do
|
39
|
+
omit 'chmod of file does not affetct root user' if Process.uid.zero?
|
40
|
+
pid_path = File.join(TMP_DIR, 'file.pid')
|
41
|
+
File.write(pid_path, '1')
|
42
|
+
|
43
|
+
mock(Process).daemon(anything, anything).never
|
44
|
+
mock(Process).kill(0, 1).once
|
45
|
+
|
46
|
+
assert_raise(Fluent::ConfigError.new('pid(1) is running')) do
|
47
|
+
Fluent::Daemonizer.daemonize(pid_path) { 'ret' }
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
test 'raise an error when file is not redable' do
|
52
|
+
omit 'chmod of file does not affetct root user' if Process.uid.zero?
|
53
|
+
not_readable_path = File.join(TMP_DIR, 'not_readable.pid')
|
54
|
+
|
55
|
+
File.write(not_readable_path, '1')
|
56
|
+
FileUtils.chmod(0333, not_readable_path)
|
57
|
+
|
58
|
+
mock(Process).daemon(anything, anything).never
|
59
|
+
assert_raise(Fluent::ConfigError.new("Cannot access pid file: #{File.absolute_path(not_readable_path)}")) do
|
60
|
+
Fluent::Daemonizer.daemonize(not_readable_path) { 'ret' }
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
test 'raise an error when file is not writable' do
|
65
|
+
omit 'chmod of file does not affetct root user' if Process.uid.zero?
|
66
|
+
not_writable_path = File.join(TMP_DIR, 'not_writable.pid')
|
67
|
+
|
68
|
+
File.write(not_writable_path, '1')
|
69
|
+
FileUtils.chmod(0555, not_writable_path)
|
70
|
+
|
71
|
+
mock(Process).daemon(anything, anything).never
|
72
|
+
assert_raise(Fluent::ConfigError.new("Cannot access pid file: #{File.absolute_path(not_writable_path)}")) do
|
73
|
+
Fluent::Daemonizer.daemonize(not_writable_path) { 'ret' }
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
test 'raise an error when directory is not writable' do
|
78
|
+
omit 'chmod of file does not affetct root user' if Process.uid.zero?
|
79
|
+
not_writable_dir = File.join(TMP_DIR, 'not_writable')
|
80
|
+
pid_path = File.join(not_writable_dir, 'file.pid')
|
81
|
+
|
82
|
+
FileUtils.mkdir_p(not_writable_dir)
|
83
|
+
FileUtils.chmod(0555, not_writable_dir)
|
84
|
+
|
85
|
+
mock(Process).daemon(anything, anything).never
|
86
|
+
assert_raise(Fluent::ConfigError.new("Cannot access directory for pid file: #{File.absolute_path(not_writable_dir)}")) do
|
87
|
+
Fluent::Daemonizer.daemonize(pid_path) { 'ret' }
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
data/test/test_log.rb
CHANGED
@@ -367,6 +367,50 @@ class LogTest < Test::Unit::TestCase
|
|
367
367
|
end
|
368
368
|
end
|
369
369
|
|
370
|
+
sub_test_case "ignore_repeated_log_interval" do
|
371
|
+
def test_same_message
|
372
|
+
message = "This is test"
|
373
|
+
logger = ServerEngine::DaemonLogger.new(@log_device, {log_level: ServerEngine::DaemonLogger::INFO})
|
374
|
+
log = Fluent::Log.new(logger, {ignore_repeated_log_interval: 5})
|
375
|
+
|
376
|
+
log.error message
|
377
|
+
10.times { |i|
|
378
|
+
Timecop.freeze(@timestamp + i)
|
379
|
+
log.error message
|
380
|
+
}
|
381
|
+
|
382
|
+
expected = [
|
383
|
+
"2016-04-21 02:58:41 +0000 [error]: This is test\n",
|
384
|
+
"2016-04-21 02:58:47 +0000 [error]: This is test\n"
|
385
|
+
]
|
386
|
+
assert_equal(expected, log.out.logs)
|
387
|
+
end
|
388
|
+
|
389
|
+
def test_different_message
|
390
|
+
message = "This is test"
|
391
|
+
logger = ServerEngine::DaemonLogger.new(@log_device, {log_level: ServerEngine::DaemonLogger::INFO})
|
392
|
+
log = Fluent::Log.new(logger, {ignore_repeated_log_interval: 10})
|
393
|
+
|
394
|
+
log.error message
|
395
|
+
3.times { |i|
|
396
|
+
Timecop.freeze(@timestamp + i)
|
397
|
+
log.error message
|
398
|
+
log.error message
|
399
|
+
log.info "Hello! " + message
|
400
|
+
}
|
401
|
+
|
402
|
+
expected = [
|
403
|
+
"2016-04-21 02:58:41 +0000 [error]: This is test\n",
|
404
|
+
"2016-04-21 02:58:41 +0000 [info]: Hello! This is test\n",
|
405
|
+
"2016-04-21 02:58:42 +0000 [error]: This is test\n",
|
406
|
+
"2016-04-21 02:58:42 +0000 [info]: Hello! This is test\n",
|
407
|
+
"2016-04-21 02:58:43 +0000 [error]: This is test\n",
|
408
|
+
"2016-04-21 02:58:43 +0000 [info]: Hello! This is test\n",
|
409
|
+
]
|
410
|
+
assert_equal(expected, log.out.logs)
|
411
|
+
end
|
412
|
+
end
|
413
|
+
|
370
414
|
def test_dup
|
371
415
|
dl_opts = {}
|
372
416
|
dl_opts[:log_level] = ServerEngine::DaemonLogger::TRACE
|
@@ -0,0 +1,18 @@
|
|
1
|
+
require_relative 'helper'
|
2
|
+
require 'fluent/msgpack_factory'
|
3
|
+
|
4
|
+
class MessagePackFactoryTest < Test::Unit::TestCase
|
5
|
+
test 'call log.warn only once' do
|
6
|
+
klass = Class.new do
|
7
|
+
include Fluent::MessagePackFactory::Mixin
|
8
|
+
end
|
9
|
+
|
10
|
+
mp = klass.new
|
11
|
+
|
12
|
+
mock.proxy($log).warn(anything).once
|
13
|
+
|
14
|
+
assert mp.msgpack_factory
|
15
|
+
assert mp.msgpack_factory
|
16
|
+
assert mp.msgpack_factory
|
17
|
+
end
|
18
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluentd
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.11.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-06-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: msgpack
|
@@ -359,6 +359,7 @@ files:
|
|
359
359
|
- ".github/ISSUE_TEMPLATE/bug_report.md"
|
360
360
|
- ".github/ISSUE_TEMPLATE/feature_request.md"
|
361
361
|
- ".github/PULL_REQUEST_TEMPLATE.md"
|
362
|
+
- ".github/workflows/issue-auto-closer.yml"
|
362
363
|
- ".gitignore"
|
363
364
|
- ".gitlab-ci.yml"
|
364
365
|
- ".travis.yml"
|
@@ -383,6 +384,7 @@ files:
|
|
383
384
|
- bin/fluent-plugin-generate
|
384
385
|
- bin/fluentd
|
385
386
|
- code-of-conduct.md
|
387
|
+
- docs/SECURITY_AUDIT.pdf
|
386
388
|
- example/copy_roundrobin.conf
|
387
389
|
- example/counter.conf
|
388
390
|
- example/filter_stdout.conf
|
@@ -475,6 +477,7 @@ files:
|
|
475
477
|
- lib/fluent/counter/store.rb
|
476
478
|
- lib/fluent/counter/validator.rb
|
477
479
|
- lib/fluent/daemon.rb
|
480
|
+
- lib/fluent/daemonizer.rb
|
478
481
|
- lib/fluent/engine.rb
|
479
482
|
- lib/fluent/env.rb
|
480
483
|
- lib/fluent/error.rb
|
@@ -576,6 +579,7 @@ files:
|
|
576
579
|
- lib/fluent/plugin/parser_syslog.rb
|
577
580
|
- lib/fluent/plugin/parser_tsv.rb
|
578
581
|
- lib/fluent/plugin/sd_file.rb
|
582
|
+
- lib/fluent/plugin/sd_srv.rb
|
579
583
|
- lib/fluent/plugin/sd_static.rb
|
580
584
|
- lib/fluent/plugin/service_discovery.rb
|
581
585
|
- lib/fluent/plugin/socket_util.rb
|
@@ -707,6 +711,8 @@ files:
|
|
707
711
|
- test/plugin/data/sd_file/config.yaml
|
708
712
|
- test/plugin/data/sd_file/config.yml
|
709
713
|
- test/plugin/data/sd_file/invalid_config.yml
|
714
|
+
- test/plugin/in_tail/test_fifo.rb
|
715
|
+
- test/plugin/in_tail/test_io_handler.rb
|
710
716
|
- test/plugin/in_tail/test_position_file.rb
|
711
717
|
- test/plugin/out_forward/test_ack_handler.rb
|
712
718
|
- test/plugin/out_forward/test_connection_manager.rb
|
@@ -789,12 +795,18 @@ files:
|
|
789
795
|
- test/plugin/test_parser_syslog.rb
|
790
796
|
- test/plugin/test_parser_tsv.rb
|
791
797
|
- test/plugin/test_sd_file.rb
|
798
|
+
- test/plugin/test_sd_srv.rb
|
792
799
|
- test/plugin/test_storage.rb
|
793
800
|
- test/plugin/test_storage_local.rb
|
794
801
|
- test/plugin/test_string_util.rb
|
795
802
|
- test/plugin_helper/data/cert/cert-key.pem
|
803
|
+
- test/plugin_helper/data/cert/cert-with-CRLF.pem
|
796
804
|
- test/plugin_helper/data/cert/cert-with-no-newline.pem
|
797
805
|
- test/plugin_helper/data/cert/cert.pem
|
806
|
+
- test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem
|
807
|
+
- test/plugin_helper/data/cert/cert_chains/ca-cert.pem
|
808
|
+
- test/plugin_helper/data/cert/cert_chains/cert-key.pem
|
809
|
+
- test/plugin_helper/data/cert/cert_chains/cert.pem
|
798
810
|
- test/plugin_helper/data/cert/generate_cert.rb
|
799
811
|
- test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem
|
800
812
|
- test/plugin_helper/data/cert/with_ca/ca-cert-key.pem
|
@@ -826,6 +838,7 @@ files:
|
|
826
838
|
- test/plugin_helper/test_retry_state.rb
|
827
839
|
- test/plugin_helper/test_server.rb
|
828
840
|
- test/plugin_helper/test_service_discovery.rb
|
841
|
+
- test/plugin_helper/test_socket.rb
|
829
842
|
- test/plugin_helper/test_storage.rb
|
830
843
|
- test/plugin_helper/test_thread.rb
|
831
844
|
- test/plugin_helper/test_timer.rb
|
@@ -839,6 +852,7 @@ files:
|
|
839
852
|
- test/test_clock.rb
|
840
853
|
- test/test_config.rb
|
841
854
|
- test/test_configdsl.rb
|
855
|
+
- test/test_daemonizer.rb
|
842
856
|
- test/test_engine.rb
|
843
857
|
- test/test_event.rb
|
844
858
|
- test/test_event_router.rb
|
@@ -851,6 +865,7 @@ files:
|
|
851
865
|
- test/test_logger_initializer.rb
|
852
866
|
- test/test_match.rb
|
853
867
|
- test/test_mixin.rb
|
868
|
+
- test/test_msgpack_factory.rb
|
854
869
|
- test/test_output.rb
|
855
870
|
- test/test_plugin.rb
|
856
871
|
- test/test_plugin_classes.rb
|
@@ -929,6 +944,8 @@ test_files:
|
|
929
944
|
- test/plugin/data/sd_file/config.yaml
|
930
945
|
- test/plugin/data/sd_file/config.yml
|
931
946
|
- test/plugin/data/sd_file/invalid_config.yml
|
947
|
+
- test/plugin/in_tail/test_fifo.rb
|
948
|
+
- test/plugin/in_tail/test_io_handler.rb
|
932
949
|
- test/plugin/in_tail/test_position_file.rb
|
933
950
|
- test/plugin/out_forward/test_ack_handler.rb
|
934
951
|
- test/plugin/out_forward/test_connection_manager.rb
|
@@ -1011,12 +1028,18 @@ test_files:
|
|
1011
1028
|
- test/plugin/test_parser_syslog.rb
|
1012
1029
|
- test/plugin/test_parser_tsv.rb
|
1013
1030
|
- test/plugin/test_sd_file.rb
|
1031
|
+
- test/plugin/test_sd_srv.rb
|
1014
1032
|
- test/plugin/test_storage.rb
|
1015
1033
|
- test/plugin/test_storage_local.rb
|
1016
1034
|
- test/plugin/test_string_util.rb
|
1017
1035
|
- test/plugin_helper/data/cert/cert-key.pem
|
1036
|
+
- test/plugin_helper/data/cert/cert-with-CRLF.pem
|
1018
1037
|
- test/plugin_helper/data/cert/cert-with-no-newline.pem
|
1019
1038
|
- test/plugin_helper/data/cert/cert.pem
|
1039
|
+
- test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem
|
1040
|
+
- test/plugin_helper/data/cert/cert_chains/ca-cert.pem
|
1041
|
+
- test/plugin_helper/data/cert/cert_chains/cert-key.pem
|
1042
|
+
- test/plugin_helper/data/cert/cert_chains/cert.pem
|
1020
1043
|
- test/plugin_helper/data/cert/generate_cert.rb
|
1021
1044
|
- test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem
|
1022
1045
|
- test/plugin_helper/data/cert/with_ca/ca-cert-key.pem
|
@@ -1048,6 +1071,7 @@ test_files:
|
|
1048
1071
|
- test/plugin_helper/test_retry_state.rb
|
1049
1072
|
- test/plugin_helper/test_server.rb
|
1050
1073
|
- test/plugin_helper/test_service_discovery.rb
|
1074
|
+
- test/plugin_helper/test_socket.rb
|
1051
1075
|
- test/plugin_helper/test_storage.rb
|
1052
1076
|
- test/plugin_helper/test_thread.rb
|
1053
1077
|
- test/plugin_helper/test_timer.rb
|
@@ -1061,6 +1085,7 @@ test_files:
|
|
1061
1085
|
- test/test_clock.rb
|
1062
1086
|
- test/test_config.rb
|
1063
1087
|
- test/test_configdsl.rb
|
1088
|
+
- test/test_daemonizer.rb
|
1064
1089
|
- test/test_engine.rb
|
1065
1090
|
- test/test_event.rb
|
1066
1091
|
- test/test_event_router.rb
|
@@ -1073,6 +1098,7 @@ test_files:
|
|
1073
1098
|
- test/test_logger_initializer.rb
|
1074
1099
|
- test/test_match.rb
|
1075
1100
|
- test/test_mixin.rb
|
1101
|
+
- test/test_msgpack_factory.rb
|
1076
1102
|
- test/test_output.rb
|
1077
1103
|
- test/test_plugin.rb
|
1078
1104
|
- test/test_plugin_classes.rb
|