fluentd 1.10.0 → 1.11.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (66) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +117 -1
  3. data/CONTRIBUTING.md +1 -1
  4. data/README.md +4 -0
  5. data/docs/SECURITY_AUDIT.pdf +0 -0
  6. data/lib/fluent/command/debug.rb +1 -0
  7. data/lib/fluent/command/fluentd.rb +14 -1
  8. data/lib/fluent/config.rb +1 -0
  9. data/lib/fluent/daemonizer.rb +88 -0
  10. data/lib/fluent/log.rb +45 -6
  11. data/lib/fluent/match.rb +1 -1
  12. data/lib/fluent/plugin/in_dummy.rb +2 -2
  13. data/lib/fluent/plugin/in_forward.rb +2 -2
  14. data/lib/fluent/plugin/in_gc_stat.rb +16 -0
  15. data/lib/fluent/plugin/in_http.rb +2 -2
  16. data/lib/fluent/plugin/in_monitor_agent.rb +1 -1
  17. data/lib/fluent/plugin/in_syslog.rb +4 -4
  18. data/lib/fluent/plugin/in_tail.rb +3 -3
  19. data/lib/fluent/plugin/in_tail/position_file.rb +23 -6
  20. data/lib/fluent/plugin/in_unix.rb +77 -77
  21. data/lib/fluent/plugin/out_copy.rb +1 -1
  22. data/lib/fluent/plugin/out_file.rb +1 -1
  23. data/lib/fluent/plugin/out_forward.rb +23 -18
  24. data/lib/fluent/plugin/out_forward/load_balancer.rb +1 -1
  25. data/lib/fluent/plugin/out_http.rb +15 -2
  26. data/lib/fluent/plugin/parser_multiline.rb +1 -1
  27. data/lib/fluent/plugin/parser_syslog.rb +216 -55
  28. data/lib/fluent/plugin_helper/record_accessor.rb +14 -0
  29. data/lib/fluent/plugin_helper/service_discovery.rb +7 -0
  30. data/lib/fluent/plugin_helper/service_discovery/manager.rb +8 -0
  31. data/lib/fluent/plugin_helper/socket.rb +20 -2
  32. data/lib/fluent/plugin_helper/socket_option.rb +2 -2
  33. data/lib/fluent/supervisor.rb +21 -9
  34. data/lib/fluent/system_config.rb +2 -1
  35. data/lib/fluent/test/filter_test.rb +2 -2
  36. data/lib/fluent/test/output_test.rb +3 -3
  37. data/lib/fluent/version.rb +1 -1
  38. data/test/command/test_fluentd.rb +57 -10
  39. data/test/config/test_system_config.rb +2 -0
  40. data/test/plugin/in_tail/test_position_file.rb +24 -0
  41. data/test/plugin/out_forward/test_load_balancer.rb +46 -0
  42. data/test/plugin/test_in_gc_stat.rb +24 -1
  43. data/test/plugin/test_in_syslog.rb +16 -1
  44. data/test/plugin/test_in_tail.rb +39 -16
  45. data/test/plugin/test_in_unix.rb +128 -73
  46. data/test/plugin/test_out_forward.rb +11 -2
  47. data/test/plugin/test_out_http.rb +38 -0
  48. data/test/plugin/test_out_null.rb +1 -1
  49. data/test/plugin/test_output_as_buffered_retries.rb +12 -4
  50. data/test/plugin/test_output_as_buffered_secondary.rb +9 -1
  51. data/test/plugin/test_parser_syslog.rb +77 -29
  52. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +27 -0
  53. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +20 -0
  54. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +27 -0
  55. data/test/plugin_helper/data/cert/cert_chains/cert.pem +40 -0
  56. data/test/plugin_helper/data/cert/generate_cert.rb +38 -0
  57. data/test/plugin_helper/http_server/test_app.rb +1 -1
  58. data/test/plugin_helper/http_server/test_route.rb +1 -1
  59. data/test/plugin_helper/test_http_server_helper.rb +2 -2
  60. data/test/plugin_helper/test_record_accessor.rb +41 -0
  61. data/test/plugin_helper/test_server.rb +1 -1
  62. data/test/plugin_helper/test_service_discovery.rb +37 -4
  63. data/test/plugin_helper/test_socket.rb +131 -0
  64. data/test/test_daemonizer.rb +91 -0
  65. data/test/test_log.rb +44 -0
  66. metadata +16 -2
@@ -0,0 +1,91 @@
1
+ require_relative 'helper'
2
+ require 'fluent/daemonizer'
3
+
4
+ class DaemonizerTest < ::Test::Unit::TestCase
5
+ TMP_DIR = File.join(File.dirname(__FILE__), 'tmp', 'daemonizer')
6
+
7
+ setup do
8
+ FileUtils.mkdir_p(TMP_DIR)
9
+ end
10
+
11
+ teardown do
12
+ FileUtils.rm_rf(TMP_DIR) rescue nil
13
+ end
14
+
15
+ test 'makes pid file' do
16
+ pid_path = File.join(TMP_DIR, 'file.pid')
17
+
18
+ mock(Process).daemon(anything, anything).once
19
+ r = Fluent::Daemonizer.daemonize(pid_path) { 'ret' }
20
+ assert_equal 'ret', r
21
+ assert File.exist?(pid_path)
22
+ assert Process.pid.to_s, File.read(pid_path).to_s
23
+ end
24
+
25
+ test 'in platforms which do not support fork' do
26
+ pid_path = File.join(TMP_DIR, 'file.pid')
27
+
28
+ mock(Process).daemon(anything, anything) { raise NotImplementedError }
29
+ args = ['-c', 'test.conf']
30
+ mock(Process).spawn(anything, *args) { Process.pid }
31
+
32
+ Fluent::Daemonizer.daemonize(pid_path, args) { 'ret' }
33
+ assert File.exist?(pid_path)
34
+ assert Process.pid.to_s, File.read(pid_path).to_s
35
+ end
36
+
37
+ sub_test_case 'when pid file already exists' do
38
+ test 'raise an error when process is running' do
39
+ omit 'chmod of file does not affetct root user' if Process.uid.zero?
40
+ pid_path = File.join(TMP_DIR, 'file.pid')
41
+ File.write(pid_path, '1')
42
+
43
+ mock(Process).daemon(anything, anything).never
44
+ mock(Process).kill(0, 1).once
45
+
46
+ assert_raise(Fluent::ConfigError.new('pid(1) is running')) do
47
+ Fluent::Daemonizer.daemonize(pid_path) { 'ret' }
48
+ end
49
+ end
50
+
51
+ test 'raise an error when file is not redable' do
52
+ omit 'chmod of file does not affetct root user' if Process.uid.zero?
53
+ not_readable_path = File.join(TMP_DIR, 'not_readable.pid')
54
+
55
+ File.write(not_readable_path, '1')
56
+ FileUtils.chmod(0333, not_readable_path)
57
+
58
+ mock(Process).daemon(anything, anything).never
59
+ assert_raise(Fluent::ConfigError.new("Cannot access pid file: #{File.absolute_path(not_readable_path)}")) do
60
+ Fluent::Daemonizer.daemonize(not_readable_path) { 'ret' }
61
+ end
62
+ end
63
+
64
+ test 'raise an error when file is not writable' do
65
+ omit 'chmod of file does not affetct root user' if Process.uid.zero?
66
+ not_writable_path = File.join(TMP_DIR, 'not_writable.pid')
67
+
68
+ File.write(not_writable_path, '1')
69
+ FileUtils.chmod(0555, not_writable_path)
70
+
71
+ mock(Process).daemon(anything, anything).never
72
+ assert_raise(Fluent::ConfigError.new("Cannot access pid file: #{File.absolute_path(not_writable_path)}")) do
73
+ Fluent::Daemonizer.daemonize(not_writable_path) { 'ret' }
74
+ end
75
+ end
76
+
77
+ test 'raise an error when directory is not writable' do
78
+ omit 'chmod of file does not affetct root user' if Process.uid.zero?
79
+ not_writable_dir = File.join(TMP_DIR, 'not_writable')
80
+ pid_path = File.join(not_writable_dir, 'file.pid')
81
+
82
+ FileUtils.mkdir_p(not_writable_dir)
83
+ FileUtils.chmod(0555, not_writable_dir)
84
+
85
+ mock(Process).daemon(anything, anything).never
86
+ assert_raise(Fluent::ConfigError.new("Cannot access directory for pid file: #{File.absolute_path(not_writable_dir)}")) do
87
+ Fluent::Daemonizer.daemonize(pid_path) { 'ret' }
88
+ end
89
+ end
90
+ end
91
+ end
@@ -367,6 +367,50 @@ class LogTest < Test::Unit::TestCase
367
367
  end
368
368
  end
369
369
 
370
+ sub_test_case "ignore_repeated_log_interval" do
371
+ def test_same_message
372
+ message = "This is test"
373
+ logger = ServerEngine::DaemonLogger.new(@log_device, {log_level: ServerEngine::DaemonLogger::INFO})
374
+ log = Fluent::Log.new(logger, {ignore_repeated_log_interval: 5})
375
+
376
+ log.error message
377
+ 10.times { |i|
378
+ Timecop.freeze(@timestamp + i)
379
+ log.error message
380
+ }
381
+
382
+ expected = [
383
+ "2016-04-21 02:58:41 +0000 [error]: This is test\n",
384
+ "2016-04-21 02:58:47 +0000 [error]: This is test\n"
385
+ ]
386
+ assert_equal(expected, log.out.logs)
387
+ end
388
+
389
+ def test_different_message
390
+ message = "This is test"
391
+ logger = ServerEngine::DaemonLogger.new(@log_device, {log_level: ServerEngine::DaemonLogger::INFO})
392
+ log = Fluent::Log.new(logger, {ignore_repeated_log_interval: 10})
393
+
394
+ log.error message
395
+ 3.times { |i|
396
+ Timecop.freeze(@timestamp + i)
397
+ log.error message
398
+ log.error message
399
+ log.info "Hello! " + message
400
+ }
401
+
402
+ expected = [
403
+ "2016-04-21 02:58:41 +0000 [error]: This is test\n",
404
+ "2016-04-21 02:58:41 +0000 [info]: Hello! This is test\n",
405
+ "2016-04-21 02:58:42 +0000 [error]: This is test\n",
406
+ "2016-04-21 02:58:42 +0000 [info]: Hello! This is test\n",
407
+ "2016-04-21 02:58:43 +0000 [error]: This is test\n",
408
+ "2016-04-21 02:58:43 +0000 [info]: Hello! This is test\n",
409
+ ]
410
+ assert_equal(expected, log.out.logs)
411
+ end
412
+ end
413
+
370
414
  def test_dup
371
415
  dl_opts = {}
372
416
  dl_opts[:log_level] = ServerEngine::DaemonLogger::TRACE
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluentd
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.10.0
4
+ version: 1.11.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-03-24 00:00:00.000000000 Z
11
+ date: 2020-06-04 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: msgpack
@@ -384,6 +384,7 @@ files:
384
384
  - bin/fluent-plugin-generate
385
385
  - bin/fluentd
386
386
  - code-of-conduct.md
387
+ - docs/SECURITY_AUDIT.pdf
387
388
  - example/copy_roundrobin.conf
388
389
  - example/counter.conf
389
390
  - example/filter_stdout.conf
@@ -476,6 +477,7 @@ files:
476
477
  - lib/fluent/counter/store.rb
477
478
  - lib/fluent/counter/validator.rb
478
479
  - lib/fluent/daemon.rb
480
+ - lib/fluent/daemonizer.rb
479
481
  - lib/fluent/engine.rb
480
482
  - lib/fluent/env.rb
481
483
  - lib/fluent/error.rb
@@ -801,6 +803,10 @@ files:
801
803
  - test/plugin_helper/data/cert/cert-with-CRLF.pem
802
804
  - test/plugin_helper/data/cert/cert-with-no-newline.pem
803
805
  - test/plugin_helper/data/cert/cert.pem
806
+ - test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem
807
+ - test/plugin_helper/data/cert/cert_chains/ca-cert.pem
808
+ - test/plugin_helper/data/cert/cert_chains/cert-key.pem
809
+ - test/plugin_helper/data/cert/cert_chains/cert.pem
804
810
  - test/plugin_helper/data/cert/generate_cert.rb
805
811
  - test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem
806
812
  - test/plugin_helper/data/cert/with_ca/ca-cert-key.pem
@@ -832,6 +838,7 @@ files:
832
838
  - test/plugin_helper/test_retry_state.rb
833
839
  - test/plugin_helper/test_server.rb
834
840
  - test/plugin_helper/test_service_discovery.rb
841
+ - test/plugin_helper/test_socket.rb
835
842
  - test/plugin_helper/test_storage.rb
836
843
  - test/plugin_helper/test_thread.rb
837
844
  - test/plugin_helper/test_timer.rb
@@ -845,6 +852,7 @@ files:
845
852
  - test/test_clock.rb
846
853
  - test/test_config.rb
847
854
  - test/test_configdsl.rb
855
+ - test/test_daemonizer.rb
848
856
  - test/test_engine.rb
849
857
  - test/test_event.rb
850
858
  - test/test_event_router.rb
@@ -1028,6 +1036,10 @@ test_files:
1028
1036
  - test/plugin_helper/data/cert/cert-with-CRLF.pem
1029
1037
  - test/plugin_helper/data/cert/cert-with-no-newline.pem
1030
1038
  - test/plugin_helper/data/cert/cert.pem
1039
+ - test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem
1040
+ - test/plugin_helper/data/cert/cert_chains/ca-cert.pem
1041
+ - test/plugin_helper/data/cert/cert_chains/cert-key.pem
1042
+ - test/plugin_helper/data/cert/cert_chains/cert.pem
1031
1043
  - test/plugin_helper/data/cert/generate_cert.rb
1032
1044
  - test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem
1033
1045
  - test/plugin_helper/data/cert/with_ca/ca-cert-key.pem
@@ -1059,6 +1071,7 @@ test_files:
1059
1071
  - test/plugin_helper/test_retry_state.rb
1060
1072
  - test/plugin_helper/test_server.rb
1061
1073
  - test/plugin_helper/test_service_discovery.rb
1074
+ - test/plugin_helper/test_socket.rb
1062
1075
  - test/plugin_helper/test_storage.rb
1063
1076
  - test/plugin_helper/test_thread.rb
1064
1077
  - test/plugin_helper/test_timer.rb
@@ -1072,6 +1085,7 @@ test_files:
1072
1085
  - test/test_clock.rb
1073
1086
  - test/test_config.rb
1074
1087
  - test/test_configdsl.rb
1088
+ - test/test_daemonizer.rb
1075
1089
  - test/test_engine.rb
1076
1090
  - test/test_event.rb
1077
1091
  - test/test_event_router.rb