fluentd 1.9.3 → 1.11.1

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (91) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/issue-auto-closer.yml +12 -0
  3. data/CHANGELOG.md +182 -0
  4. data/CONTRIBUTING.md +1 -1
  5. data/README.md +4 -0
  6. data/docs/SECURITY_AUDIT.pdf +0 -0
  7. data/lib/fluent/command/debug.rb +1 -0
  8. data/lib/fluent/command/fluentd.rb +25 -1
  9. data/lib/fluent/config.rb +1 -0
  10. data/lib/fluent/daemonizer.rb +88 -0
  11. data/lib/fluent/log.rb +45 -6
  12. data/lib/fluent/match.rb +1 -1
  13. data/lib/fluent/msgpack_factory.rb +13 -6
  14. data/lib/fluent/plugin/buffer.rb +2 -2
  15. data/lib/fluent/plugin/in_dummy.rb +3 -3
  16. data/lib/fluent/plugin/in_forward.rb +2 -2
  17. data/lib/fluent/plugin/in_gc_stat.rb +16 -0
  18. data/lib/fluent/plugin/in_http.rb +146 -75
  19. data/lib/fluent/plugin/in_monitor_agent.rb +1 -1
  20. data/lib/fluent/plugin/in_syslog.rb +4 -4
  21. data/lib/fluent/plugin/in_tail.rb +40 -31
  22. data/lib/fluent/plugin/in_tail/position_file.rb +23 -6
  23. data/lib/fluent/plugin/in_unix.rb +77 -77
  24. data/lib/fluent/plugin/out_copy.rb +1 -1
  25. data/lib/fluent/plugin/out_file.rb +1 -1
  26. data/lib/fluent/plugin/out_forward.rb +25 -22
  27. data/lib/fluent/plugin/out_forward/handshake_protocol.rb +4 -0
  28. data/lib/fluent/plugin/out_forward/load_balancer.rb +1 -1
  29. data/lib/fluent/plugin/out_http.rb +15 -2
  30. data/lib/fluent/plugin/parser_multiline.rb +1 -1
  31. data/lib/fluent/plugin/parser_syslog.rb +303 -62
  32. data/lib/fluent/plugin/sd_file.rb +1 -0
  33. data/lib/fluent/plugin/sd_srv.rb +135 -0
  34. data/lib/fluent/plugin_helper/cert_option.rb +15 -2
  35. data/lib/fluent/plugin_helper/child_process.rb +3 -2
  36. data/lib/fluent/plugin_helper/record_accessor.rb +14 -0
  37. data/lib/fluent/plugin_helper/server.rb +3 -1
  38. data/lib/fluent/plugin_helper/service_discovery.rb +7 -0
  39. data/lib/fluent/plugin_helper/service_discovery/manager.rb +8 -0
  40. data/lib/fluent/plugin_helper/socket.rb +20 -2
  41. data/lib/fluent/plugin_helper/socket_option.rb +21 -3
  42. data/lib/fluent/supervisor.rb +21 -9
  43. data/lib/fluent/system_config.rb +2 -1
  44. data/lib/fluent/test/filter_test.rb +2 -2
  45. data/lib/fluent/test/output_test.rb +3 -3
  46. data/lib/fluent/version.rb +1 -1
  47. data/test/command/test_fluentd.rb +71 -12
  48. data/test/config/test_system_config.rb +2 -0
  49. data/test/helper.rb +2 -2
  50. data/test/plugin/in_tail/test_fifo.rb +121 -0
  51. data/test/plugin/in_tail/test_io_handler.rb +132 -0
  52. data/test/plugin/in_tail/test_position_file.rb +25 -1
  53. data/test/plugin/out_forward/test_handshake_protocol.rb +10 -1
  54. data/test/plugin/out_forward/test_load_balancer.rb +46 -0
  55. data/test/plugin/test_buf_file.rb +3 -1
  56. data/test/plugin/test_buffer.rb +20 -0
  57. data/test/plugin/test_compressable.rb +7 -4
  58. data/test/plugin/test_in_dummy.rb +12 -14
  59. data/test/plugin/test_in_forward.rb +2 -2
  60. data/test/plugin/test_in_gc_stat.rb +24 -1
  61. data/test/plugin/test_in_http.rb +57 -0
  62. data/test/plugin/test_in_syslog.rb +16 -1
  63. data/test/plugin/test_in_tail.rb +43 -20
  64. data/test/plugin/test_in_unix.rb +128 -73
  65. data/test/plugin/test_out_forward.rb +39 -3
  66. data/test/plugin/test_out_http.rb +38 -0
  67. data/test/plugin/test_out_null.rb +1 -1
  68. data/test/plugin/test_output_as_buffered_retries.rb +12 -4
  69. data/test/plugin/test_output_as_buffered_secondary.rb +9 -1
  70. data/test/plugin/test_parser_syslog.rb +106 -46
  71. data/test/plugin/test_sd_file.rb +17 -0
  72. data/test/plugin/test_sd_srv.rb +230 -0
  73. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +19 -0
  74. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +27 -0
  75. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +20 -0
  76. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +27 -0
  77. data/test/plugin_helper/data/cert/cert_chains/cert.pem +40 -0
  78. data/test/plugin_helper/data/cert/generate_cert.rb +38 -0
  79. data/test/plugin_helper/http_server/test_app.rb +1 -1
  80. data/test/plugin_helper/http_server/test_route.rb +1 -1
  81. data/test/plugin_helper/test_cert_option.rb +2 -0
  82. data/test/plugin_helper/test_child_process.rb +20 -3
  83. data/test/plugin_helper/test_http_server_helper.rb +2 -2
  84. data/test/plugin_helper/test_record_accessor.rb +41 -0
  85. data/test/plugin_helper/test_server.rb +1 -1
  86. data/test/plugin_helper/test_service_discovery.rb +37 -4
  87. data/test/plugin_helper/test_socket.rb +131 -0
  88. data/test/test_daemonizer.rb +91 -0
  89. data/test/test_log.rb +44 -0
  90. data/test/test_msgpack_factory.rb +18 -0
  91. metadata +28 -2
@@ -0,0 +1,91 @@
1
+ require_relative 'helper'
2
+ require 'fluent/daemonizer'
3
+
4
+ class DaemonizerTest < ::Test::Unit::TestCase
5
+ TMP_DIR = File.join(File.dirname(__FILE__), 'tmp', 'daemonizer')
6
+
7
+ setup do
8
+ FileUtils.mkdir_p(TMP_DIR)
9
+ end
10
+
11
+ teardown do
12
+ FileUtils.rm_rf(TMP_DIR) rescue nil
13
+ end
14
+
15
+ test 'makes pid file' do
16
+ pid_path = File.join(TMP_DIR, 'file.pid')
17
+
18
+ mock(Process).daemon(anything, anything).once
19
+ r = Fluent::Daemonizer.daemonize(pid_path) { 'ret' }
20
+ assert_equal 'ret', r
21
+ assert File.exist?(pid_path)
22
+ assert Process.pid.to_s, File.read(pid_path).to_s
23
+ end
24
+
25
+ test 'in platforms which do not support fork' do
26
+ pid_path = File.join(TMP_DIR, 'file.pid')
27
+
28
+ mock(Process).daemon(anything, anything) { raise NotImplementedError }
29
+ args = ['-c', 'test.conf']
30
+ mock(Process).spawn(anything, *args) { Process.pid }
31
+
32
+ Fluent::Daemonizer.daemonize(pid_path, args) { 'ret' }
33
+ assert File.exist?(pid_path)
34
+ assert Process.pid.to_s, File.read(pid_path).to_s
35
+ end
36
+
37
+ sub_test_case 'when pid file already exists' do
38
+ test 'raise an error when process is running' do
39
+ omit 'chmod of file does not affetct root user' if Process.uid.zero?
40
+ pid_path = File.join(TMP_DIR, 'file.pid')
41
+ File.write(pid_path, '1')
42
+
43
+ mock(Process).daemon(anything, anything).never
44
+ mock(Process).kill(0, 1).once
45
+
46
+ assert_raise(Fluent::ConfigError.new('pid(1) is running')) do
47
+ Fluent::Daemonizer.daemonize(pid_path) { 'ret' }
48
+ end
49
+ end
50
+
51
+ test 'raise an error when file is not redable' do
52
+ omit 'chmod of file does not affetct root user' if Process.uid.zero?
53
+ not_readable_path = File.join(TMP_DIR, 'not_readable.pid')
54
+
55
+ File.write(not_readable_path, '1')
56
+ FileUtils.chmod(0333, not_readable_path)
57
+
58
+ mock(Process).daemon(anything, anything).never
59
+ assert_raise(Fluent::ConfigError.new("Cannot access pid file: #{File.absolute_path(not_readable_path)}")) do
60
+ Fluent::Daemonizer.daemonize(not_readable_path) { 'ret' }
61
+ end
62
+ end
63
+
64
+ test 'raise an error when file is not writable' do
65
+ omit 'chmod of file does not affetct root user' if Process.uid.zero?
66
+ not_writable_path = File.join(TMP_DIR, 'not_writable.pid')
67
+
68
+ File.write(not_writable_path, '1')
69
+ FileUtils.chmod(0555, not_writable_path)
70
+
71
+ mock(Process).daemon(anything, anything).never
72
+ assert_raise(Fluent::ConfigError.new("Cannot access pid file: #{File.absolute_path(not_writable_path)}")) do
73
+ Fluent::Daemonizer.daemonize(not_writable_path) { 'ret' }
74
+ end
75
+ end
76
+
77
+ test 'raise an error when directory is not writable' do
78
+ omit 'chmod of file does not affetct root user' if Process.uid.zero?
79
+ not_writable_dir = File.join(TMP_DIR, 'not_writable')
80
+ pid_path = File.join(not_writable_dir, 'file.pid')
81
+
82
+ FileUtils.mkdir_p(not_writable_dir)
83
+ FileUtils.chmod(0555, not_writable_dir)
84
+
85
+ mock(Process).daemon(anything, anything).never
86
+ assert_raise(Fluent::ConfigError.new("Cannot access directory for pid file: #{File.absolute_path(not_writable_dir)}")) do
87
+ Fluent::Daemonizer.daemonize(pid_path) { 'ret' }
88
+ end
89
+ end
90
+ end
91
+ end
@@ -367,6 +367,50 @@ class LogTest < Test::Unit::TestCase
367
367
  end
368
368
  end
369
369
 
370
+ sub_test_case "ignore_repeated_log_interval" do
371
+ def test_same_message
372
+ message = "This is test"
373
+ logger = ServerEngine::DaemonLogger.new(@log_device, {log_level: ServerEngine::DaemonLogger::INFO})
374
+ log = Fluent::Log.new(logger, {ignore_repeated_log_interval: 5})
375
+
376
+ log.error message
377
+ 10.times { |i|
378
+ Timecop.freeze(@timestamp + i)
379
+ log.error message
380
+ }
381
+
382
+ expected = [
383
+ "2016-04-21 02:58:41 +0000 [error]: This is test\n",
384
+ "2016-04-21 02:58:47 +0000 [error]: This is test\n"
385
+ ]
386
+ assert_equal(expected, log.out.logs)
387
+ end
388
+
389
+ def test_different_message
390
+ message = "This is test"
391
+ logger = ServerEngine::DaemonLogger.new(@log_device, {log_level: ServerEngine::DaemonLogger::INFO})
392
+ log = Fluent::Log.new(logger, {ignore_repeated_log_interval: 10})
393
+
394
+ log.error message
395
+ 3.times { |i|
396
+ Timecop.freeze(@timestamp + i)
397
+ log.error message
398
+ log.error message
399
+ log.info "Hello! " + message
400
+ }
401
+
402
+ expected = [
403
+ "2016-04-21 02:58:41 +0000 [error]: This is test\n",
404
+ "2016-04-21 02:58:41 +0000 [info]: Hello! This is test\n",
405
+ "2016-04-21 02:58:42 +0000 [error]: This is test\n",
406
+ "2016-04-21 02:58:42 +0000 [info]: Hello! This is test\n",
407
+ "2016-04-21 02:58:43 +0000 [error]: This is test\n",
408
+ "2016-04-21 02:58:43 +0000 [info]: Hello! This is test\n",
409
+ ]
410
+ assert_equal(expected, log.out.logs)
411
+ end
412
+ end
413
+
370
414
  def test_dup
371
415
  dl_opts = {}
372
416
  dl_opts[:log_level] = ServerEngine::DaemonLogger::TRACE
@@ -0,0 +1,18 @@
1
+ require_relative 'helper'
2
+ require 'fluent/msgpack_factory'
3
+
4
+ class MessagePackFactoryTest < Test::Unit::TestCase
5
+ test 'call log.warn only once' do
6
+ klass = Class.new do
7
+ include Fluent::MessagePackFactory::Mixin
8
+ end
9
+
10
+ mp = klass.new
11
+
12
+ mock.proxy($log).warn(anything).once
13
+
14
+ assert mp.msgpack_factory
15
+ assert mp.msgpack_factory
16
+ assert mp.msgpack_factory
17
+ end
18
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluentd
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.9.3
4
+ version: 1.11.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-03-05 00:00:00.000000000 Z
11
+ date: 2020-06-22 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: msgpack
@@ -359,6 +359,7 @@ files:
359
359
  - ".github/ISSUE_TEMPLATE/bug_report.md"
360
360
  - ".github/ISSUE_TEMPLATE/feature_request.md"
361
361
  - ".github/PULL_REQUEST_TEMPLATE.md"
362
+ - ".github/workflows/issue-auto-closer.yml"
362
363
  - ".gitignore"
363
364
  - ".gitlab-ci.yml"
364
365
  - ".travis.yml"
@@ -383,6 +384,7 @@ files:
383
384
  - bin/fluent-plugin-generate
384
385
  - bin/fluentd
385
386
  - code-of-conduct.md
387
+ - docs/SECURITY_AUDIT.pdf
386
388
  - example/copy_roundrobin.conf
387
389
  - example/counter.conf
388
390
  - example/filter_stdout.conf
@@ -475,6 +477,7 @@ files:
475
477
  - lib/fluent/counter/store.rb
476
478
  - lib/fluent/counter/validator.rb
477
479
  - lib/fluent/daemon.rb
480
+ - lib/fluent/daemonizer.rb
478
481
  - lib/fluent/engine.rb
479
482
  - lib/fluent/env.rb
480
483
  - lib/fluent/error.rb
@@ -576,6 +579,7 @@ files:
576
579
  - lib/fluent/plugin/parser_syslog.rb
577
580
  - lib/fluent/plugin/parser_tsv.rb
578
581
  - lib/fluent/plugin/sd_file.rb
582
+ - lib/fluent/plugin/sd_srv.rb
579
583
  - lib/fluent/plugin/sd_static.rb
580
584
  - lib/fluent/plugin/service_discovery.rb
581
585
  - lib/fluent/plugin/socket_util.rb
@@ -707,6 +711,8 @@ files:
707
711
  - test/plugin/data/sd_file/config.yaml
708
712
  - test/plugin/data/sd_file/config.yml
709
713
  - test/plugin/data/sd_file/invalid_config.yml
714
+ - test/plugin/in_tail/test_fifo.rb
715
+ - test/plugin/in_tail/test_io_handler.rb
710
716
  - test/plugin/in_tail/test_position_file.rb
711
717
  - test/plugin/out_forward/test_ack_handler.rb
712
718
  - test/plugin/out_forward/test_connection_manager.rb
@@ -789,12 +795,18 @@ files:
789
795
  - test/plugin/test_parser_syslog.rb
790
796
  - test/plugin/test_parser_tsv.rb
791
797
  - test/plugin/test_sd_file.rb
798
+ - test/plugin/test_sd_srv.rb
792
799
  - test/plugin/test_storage.rb
793
800
  - test/plugin/test_storage_local.rb
794
801
  - test/plugin/test_string_util.rb
795
802
  - test/plugin_helper/data/cert/cert-key.pem
803
+ - test/plugin_helper/data/cert/cert-with-CRLF.pem
796
804
  - test/plugin_helper/data/cert/cert-with-no-newline.pem
797
805
  - test/plugin_helper/data/cert/cert.pem
806
+ - test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem
807
+ - test/plugin_helper/data/cert/cert_chains/ca-cert.pem
808
+ - test/plugin_helper/data/cert/cert_chains/cert-key.pem
809
+ - test/plugin_helper/data/cert/cert_chains/cert.pem
798
810
  - test/plugin_helper/data/cert/generate_cert.rb
799
811
  - test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem
800
812
  - test/plugin_helper/data/cert/with_ca/ca-cert-key.pem
@@ -826,6 +838,7 @@ files:
826
838
  - test/plugin_helper/test_retry_state.rb
827
839
  - test/plugin_helper/test_server.rb
828
840
  - test/plugin_helper/test_service_discovery.rb
841
+ - test/plugin_helper/test_socket.rb
829
842
  - test/plugin_helper/test_storage.rb
830
843
  - test/plugin_helper/test_thread.rb
831
844
  - test/plugin_helper/test_timer.rb
@@ -839,6 +852,7 @@ files:
839
852
  - test/test_clock.rb
840
853
  - test/test_config.rb
841
854
  - test/test_configdsl.rb
855
+ - test/test_daemonizer.rb
842
856
  - test/test_engine.rb
843
857
  - test/test_event.rb
844
858
  - test/test_event_router.rb
@@ -851,6 +865,7 @@ files:
851
865
  - test/test_logger_initializer.rb
852
866
  - test/test_match.rb
853
867
  - test/test_mixin.rb
868
+ - test/test_msgpack_factory.rb
854
869
  - test/test_output.rb
855
870
  - test/test_plugin.rb
856
871
  - test/test_plugin_classes.rb
@@ -929,6 +944,8 @@ test_files:
929
944
  - test/plugin/data/sd_file/config.yaml
930
945
  - test/plugin/data/sd_file/config.yml
931
946
  - test/plugin/data/sd_file/invalid_config.yml
947
+ - test/plugin/in_tail/test_fifo.rb
948
+ - test/plugin/in_tail/test_io_handler.rb
932
949
  - test/plugin/in_tail/test_position_file.rb
933
950
  - test/plugin/out_forward/test_ack_handler.rb
934
951
  - test/plugin/out_forward/test_connection_manager.rb
@@ -1011,12 +1028,18 @@ test_files:
1011
1028
  - test/plugin/test_parser_syslog.rb
1012
1029
  - test/plugin/test_parser_tsv.rb
1013
1030
  - test/plugin/test_sd_file.rb
1031
+ - test/plugin/test_sd_srv.rb
1014
1032
  - test/plugin/test_storage.rb
1015
1033
  - test/plugin/test_storage_local.rb
1016
1034
  - test/plugin/test_string_util.rb
1017
1035
  - test/plugin_helper/data/cert/cert-key.pem
1036
+ - test/plugin_helper/data/cert/cert-with-CRLF.pem
1018
1037
  - test/plugin_helper/data/cert/cert-with-no-newline.pem
1019
1038
  - test/plugin_helper/data/cert/cert.pem
1039
+ - test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem
1040
+ - test/plugin_helper/data/cert/cert_chains/ca-cert.pem
1041
+ - test/plugin_helper/data/cert/cert_chains/cert-key.pem
1042
+ - test/plugin_helper/data/cert/cert_chains/cert.pem
1020
1043
  - test/plugin_helper/data/cert/generate_cert.rb
1021
1044
  - test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem
1022
1045
  - test/plugin_helper/data/cert/with_ca/ca-cert-key.pem
@@ -1048,6 +1071,7 @@ test_files:
1048
1071
  - test/plugin_helper/test_retry_state.rb
1049
1072
  - test/plugin_helper/test_server.rb
1050
1073
  - test/plugin_helper/test_service_discovery.rb
1074
+ - test/plugin_helper/test_socket.rb
1051
1075
  - test/plugin_helper/test_storage.rb
1052
1076
  - test/plugin_helper/test_thread.rb
1053
1077
  - test/plugin_helper/test_timer.rb
@@ -1061,6 +1085,7 @@ test_files:
1061
1085
  - test/test_clock.rb
1062
1086
  - test/test_config.rb
1063
1087
  - test/test_configdsl.rb
1088
+ - test/test_daemonizer.rb
1064
1089
  - test/test_engine.rb
1065
1090
  - test/test_event.rb
1066
1091
  - test/test_event_router.rb
@@ -1073,6 +1098,7 @@ test_files:
1073
1098
  - test/test_logger_initializer.rb
1074
1099
  - test/test_match.rb
1075
1100
  - test/test_mixin.rb
1101
+ - test/test_msgpack_factory.rb
1076
1102
  - test/test_output.rb
1077
1103
  - test/test_plugin.rb
1078
1104
  - test/test_plugin_classes.rb