fluentd 0.14.13-x86-mingw32 → 0.14.17-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (67) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +4 -5
  3. data/ChangeLog +106 -0
  4. data/MAINTAINERS.md +5 -0
  5. data/README.md +25 -0
  6. data/example/worker_section.conf +36 -0
  7. data/fluentd.gemspec +1 -1
  8. data/lib/fluent/agent.rb +5 -2
  9. data/lib/fluent/command/binlog_reader.rb +1 -0
  10. data/lib/fluent/command/fluentd.rb +28 -12
  11. data/lib/fluent/command/plugin_config_formatter.rb +0 -1
  12. data/lib/fluent/command/plugin_generator.rb +1 -1
  13. data/lib/fluent/compat/detach_process_mixin.rb +8 -0
  14. data/lib/fluent/compat/input.rb +0 -10
  15. data/lib/fluent/compat/output.rb +0 -10
  16. data/lib/fluent/config/element.rb +22 -0
  17. data/lib/fluent/config/literal_parser.rb +2 -0
  18. data/lib/fluent/config/types.rb +2 -2
  19. data/lib/fluent/engine.rb +27 -10
  20. data/lib/fluent/env.rb +3 -3
  21. data/lib/fluent/log.rb +4 -1
  22. data/lib/fluent/plugin/base.rb +3 -0
  23. data/lib/fluent/plugin/filter.rb +2 -2
  24. data/lib/fluent/plugin/filter_parser.rb +17 -6
  25. data/lib/fluent/plugin/in_forward.rb +1 -1
  26. data/lib/fluent/plugin/in_http.rb +4 -0
  27. data/lib/fluent/plugin/in_monitor_agent.rb +8 -3
  28. data/lib/fluent/plugin/in_syslog.rb +3 -2
  29. data/lib/fluent/plugin/in_tail.rb +14 -3
  30. data/lib/fluent/plugin/in_udp.rb +6 -2
  31. data/lib/fluent/plugin/out_file.rb +5 -0
  32. data/lib/fluent/plugin/out_forward.rb +5 -2
  33. data/lib/fluent/plugin/output.rb +13 -8
  34. data/lib/fluent/plugin/parser_apache2.rb +1 -1
  35. data/lib/fluent/plugin/parser_syslog.rb +40 -1
  36. data/lib/fluent/plugin_helper/cert_option.rb +2 -2
  37. data/lib/fluent/plugin_helper/compat_parameters.rb +1 -1
  38. data/lib/fluent/plugin_helper/storage.rb +1 -1
  39. data/lib/fluent/root_agent.rb +36 -4
  40. data/lib/fluent/supervisor.rb +37 -6
  41. data/lib/fluent/system_config.rb +7 -0
  42. data/lib/fluent/time.rb +1 -0
  43. data/lib/fluent/version.rb +1 -1
  44. data/lib/fluent/winsvc.rb +25 -11
  45. data/test/command/test_fluentd.rb +253 -4
  46. data/test/config/test_element.rb +63 -0
  47. data/test/config/test_literal_parser.rb +1 -1
  48. data/test/config/test_system_config.rb +36 -6
  49. data/test/config/test_types.rb +19 -0
  50. data/test/plugin/test_filter_parser.rb +35 -0
  51. data/test/plugin/test_in_http.rb +58 -4
  52. data/test/plugin/test_in_monitor_agent.rb +90 -9
  53. data/test/plugin/test_in_tail.rb +16 -0
  54. data/test/plugin/test_in_udp.rb +11 -1
  55. data/test/plugin/test_out_file.rb +9 -0
  56. data/test/plugin/test_out_forward.rb +45 -0
  57. data/test/plugin/test_output.rb +15 -15
  58. data/test/plugin/test_output_as_buffered.rb +30 -2
  59. data/test/plugin/test_parser_apache2.rb +8 -0
  60. data/test/plugin/test_parser_syslog.rb +176 -0
  61. data/test/plugin_helper/test_server.rb +37 -31
  62. data/test/plugin_helper/test_storage.rb +9 -0
  63. data/test/test_log.rb +6 -0
  64. data/test/test_plugin_classes.rb +50 -0
  65. data/test/test_root_agent.rb +245 -14
  66. data/test/test_time_parser.rb +12 -0
  67. metadata +13 -5
@@ -297,7 +297,7 @@ CONF
297
297
  assert_log_matches(
298
298
  create_cmdline(conf_path),
299
299
  "fluentd worker is now running",
300
- '[warn]: match for some tags of log events are not defined (to be ignored) tags=["fluent.trace", "fluent.debug", "fluent.info"]',
300
+ '[warn]: #0 match for some tags of log events are not defined (to be ignored) tags=["fluent.trace", "fluent.debug", "fluent.info"]',
301
301
  '[warn]: #0 no patterns matched tag="fluent.info"',
302
302
  )
303
303
  end
@@ -334,7 +334,7 @@ CONF
334
334
  assert_log_matches(
335
335
  create_cmdline(conf_path),
336
336
  "fluentd worker is now running",
337
- '[warn]: match for some tags of log events are not defined (to be ignored) tags=["fluent.info", "fluent.fatal"]',
337
+ '[warn]: #0 match for some tags of log events are not defined (to be ignored) tags=["fluent.info", "fluent.fatal"]',
338
338
  patterns_not_match: ['[warn]: no patterns matched tag="fluent.info"'],
339
339
  )
340
340
  end
@@ -442,8 +442,7 @@ CONF
442
442
 
443
443
  assert_fluentd_fails_to_start(
444
444
  create_cmdline(conf_path, "-p", File.dirname(plugin_path)),
445
- "error_class=SyntaxError",
446
- "in_buggy.rb:5: syntax error, unexpected end-of-input, expecting keyword_end",
445
+ "in_buggy.rb:5: syntax error, unexpected end-of-input, expecting keyword_end (SyntaxError)"
447
446
  )
448
447
  end
449
448
  end
@@ -614,5 +613,255 @@ CONF
614
613
  "config error file=\"#{conf_path}\" error_class=Fluent::ConfigError error=\"Plugin 'file' does not support multi workers configuration (Fluent::Plugin::FileBuffer)\"",
615
614
  )
616
615
  end
616
+
617
+ test 'failed to start workers when configured plugins as chidren of MultiOutput do not support multi worker configuration' do
618
+ script = <<-EOC
619
+ require 'fluent/plugin/output'
620
+ module Fluent::Plugin
621
+ class SingleOutput < Output
622
+ Fluent::Plugin.register_output('single', self)
623
+ def multi_workers_ready?
624
+ false
625
+ end
626
+ def write(chunk)
627
+ end
628
+ end
629
+ end
630
+ EOC
631
+ plugin_path = create_plugin_file('out_single.rb', script)
632
+
633
+ conf = <<CONF
634
+ <system>
635
+ workers 2
636
+ </system>
637
+ <source>
638
+ @type single
639
+ @id single
640
+ @label @dummydata
641
+ </source>
642
+ <label @dummydata>
643
+ <match dummy>
644
+ @type copy
645
+ <store>
646
+ @type single
647
+ </store>
648
+ <store>
649
+ @type single
650
+ </store>
651
+ </match>
652
+ </label>
653
+ CONF
654
+ conf_path = create_conf_file('workers_invalid3.conf', conf)
655
+ assert_fluentd_fails_to_start(
656
+ create_cmdline(conf_path, "-p", File.dirname(plugin_path)),
657
+ "Plugin 'single' does not support multi workers configuration (Fluent::Plugin::SingleOutput)",
658
+ )
659
+ end
660
+
661
+ test 'success to start a worker with worker specific configuration' do
662
+ conf = <<CONF
663
+ <system>
664
+ workers 2
665
+ root_dir #{@root_path}
666
+ </system>
667
+ <source>
668
+ @type dummy
669
+ @id dummy
670
+ @label @dummydata
671
+ tag dummy
672
+ dummy {"message": "yay!"}
673
+ </source>
674
+ <worker 1>
675
+ <source>
676
+ @type dummy
677
+ @id dummy_in_worker
678
+ @label @dummydata
679
+ tag dummy
680
+ dummy {"message": "yay!"}
681
+ </source>
682
+ </worker>
683
+ <label @dummydata>
684
+ <match dummy>
685
+ @type null
686
+ @id blackhole
687
+ </match>
688
+ </label>
689
+ CONF
690
+ conf_path = create_conf_file('worker_section0.conf', conf)
691
+ assert Dir.exist?(@root_path)
692
+
693
+ assert_log_matches(
694
+ create_cmdline(conf_path),
695
+ "#0 fluentd worker is now running worker=0",
696
+ "#1 fluentd worker is now running worker=1",
697
+ /(?!#\d) adding source type="dummy"/,
698
+ '#1 adding source type="dummy"'
699
+ )
700
+ end
701
+
702
+ test 'success to start workers when configured plugins only for specific worker do not support multi worker configuration' do
703
+ script = <<-EOC
704
+ require 'fluent/plugin/input'
705
+ module Fluent::Plugin
706
+ class SingleInput < Input
707
+ Fluent::Plugin.register_input('single', self)
708
+ def multi_workers_ready?
709
+ false
710
+ end
711
+ end
712
+ end
713
+ EOC
714
+ plugin_path = create_plugin_file('in_single.rb', script)
715
+
716
+ conf = <<CONF
717
+ <system>
718
+ workers 2
719
+ </system>
720
+ <worker 1>
721
+ <source>
722
+ @type single
723
+ @id single
724
+ @label @dummydata
725
+ </source>
726
+ </worker>
727
+ <label @dummydata>
728
+ <match dummy>
729
+ @type null
730
+ @id blackhole
731
+ </match>
732
+ </label>
733
+ CONF
734
+ conf_path = create_conf_file('worker_section1.conf', conf)
735
+ assert Dir.exist?(@root_path)
736
+
737
+ assert_log_matches(
738
+ create_cmdline(conf_path, "-p", File.dirname(plugin_path)),
739
+ "#0 fluentd worker is now running worker=0",
740
+ "#1 fluentd worker is now running worker=1",
741
+ '#1 adding source type="single"'
742
+ )
743
+ end
744
+
745
+ test 'success to start workers when file buffer is configured in non-workers way only for specific worker' do
746
+ conf = <<CONF
747
+ <system>
748
+ workers 2
749
+ </system>
750
+ <source>
751
+ @type dummy
752
+ @id dummy
753
+ tag dummy
754
+ dummy {"message": "yay!"}
755
+ </source>
756
+ <worker 1>
757
+ <match dummy>
758
+ @type null
759
+ @id blackhole
760
+ <buffer>
761
+ @type file
762
+ path #{File.join(@root_path, "buf", "file.*.log")}
763
+ </buffer>
764
+ </match>
765
+ </worker>
766
+ CONF
767
+ conf_path = create_conf_file('worker_section2.conf', conf)
768
+ assert_log_matches(
769
+ create_cmdline(conf_path),
770
+ "#0 fluentd worker is now running worker=0",
771
+ "#1 fluentd worker is now running worker=1",
772
+ '#1 adding match pattern="dummy" type="null"'
773
+ )
774
+ end
775
+
776
+ test 'success to start workers when configured plugins as a chidren of MultiOutput only for specific worker do not support multi worker configuration' do
777
+ script = <<-EOC
778
+ require 'fluent/plugin/output'
779
+ module Fluent::Plugin
780
+ class SingleOutput < Output
781
+ Fluent::Plugin.register_output('single', self)
782
+ def multi_workers_ready?
783
+ false
784
+ end
785
+ def write(chunk)
786
+ end
787
+ end
788
+ end
789
+ EOC
790
+ plugin_path = create_plugin_file('out_single.rb', script)
791
+
792
+ conf = <<CONF
793
+ <system>
794
+ workers 2
795
+ </system>
796
+ <source>
797
+ @type dummy
798
+ @id dummy
799
+ tag dummy
800
+ dummy {"message": "yay!"}
801
+ </source>
802
+ <worker 1>
803
+ <match dummy>
804
+ @type copy
805
+ <store>
806
+ @type single
807
+ </store>
808
+ <store>
809
+ @type single
810
+ </store>
811
+ </match>
812
+ </worker>
813
+ CONF
814
+ conf_path = create_conf_file('worker_section3.conf', conf)
815
+ assert_log_matches(
816
+ create_cmdline(conf_path, "-p", File.dirname(plugin_path)),
817
+ "#0 fluentd worker is now running worker=0",
818
+ "#1 fluentd worker is now running worker=1",
819
+ '#1 adding match pattern="dummy" type="copy"'
820
+ )
821
+ end
822
+ end
823
+
824
+ sub_test_case 'config dump' do
825
+ test 'all secret parameters in worker section is sealed' do
826
+ script = <<-EOC
827
+ require 'fluent/plugin/input'
828
+ module Fluent::Plugin
829
+ class FakeInput < Input
830
+ Fluent::Plugin.register_input('fake', self)
831
+ config_param :secret, :string, secret: true
832
+ end
833
+ end
834
+ EOC
835
+ plugin_path = create_plugin_file('in_fake.rb', script)
836
+
837
+ conf = <<CONF
838
+ <system>
839
+ workers 2
840
+ </system>
841
+ <worker 0>
842
+ <source>
843
+ @type fake
844
+ secret secret0
845
+ </source>
846
+ <match>
847
+ @type null
848
+ </match>
849
+ </worker>
850
+ <worker 1>
851
+ <source>
852
+ @type fake
853
+ secret secret1
854
+ </source>
855
+ <match>
856
+ @type null
857
+ </match>
858
+ </worker>
859
+ CONF
860
+ conf_path = create_conf_file('secret_in_worker.conf', conf)
861
+ assert File.exist?(conf_path)
862
+
863
+ assert_log_matches(create_cmdline(conf_path, "-p", File.dirname(plugin_path)),
864
+ "secret xxxxxx", patterns_not_match: ["secret secret0", "secret secret1"])
865
+ end
617
866
  end
618
867
  end
@@ -400,4 +400,67 @@ CONF
400
400
  end
401
401
  end
402
402
  end
403
+
404
+ sub_test_case '#set_target_worker' do
405
+ test 'set target_worker_id recursively' do
406
+ e = element('label', '@mytest', {}, [ element('filter', '**'), element('match', '**', {}, [ element('store'), element('store') ]) ])
407
+ e.set_target_worker_id(1)
408
+ assert_equal 1, e.target_worker_id
409
+ assert_equal 1, e.elements[0].target_worker_id
410
+ assert_equal 1, e.elements[1].target_worker_id
411
+ assert_equal 1, e.elements[1].elements[0].target_worker_id
412
+ assert_equal 1, e.elements[1].elements[1].target_worker_id
413
+ end
414
+ end
415
+
416
+ sub_test_case '#for_every_workers?' do
417
+ test 'has target_worker_id' do
418
+ e = element()
419
+ e.set_target_worker_id(1)
420
+ assert_false e.for_every_workers?
421
+ end
422
+
423
+ test "doesn't have target_worker_id" do
424
+ e = element()
425
+ assert e.for_every_workers?
426
+ end
427
+ end
428
+
429
+ sub_test_case '#for_this_workers?' do
430
+ test 'target_worker_id == current worker_id' do
431
+ e = element()
432
+ e.set_target_worker_id(0)
433
+ assert e.for_this_worker?
434
+ end
435
+
436
+ test 'target_worker_id != current worker_id' do
437
+ e = element()
438
+ e.set_target_worker_id(1)
439
+ assert_false e.for_this_worker?
440
+ end
441
+
442
+ test "doesn't have target_worker_id" do
443
+ e = element()
444
+ assert_false e.for_this_worker?
445
+ end
446
+ end
447
+
448
+ sub_test_case '#for_another_worker?' do
449
+ test 'target_worker_id == current worker_id' do
450
+ e = element()
451
+ e.set_target_worker_id(0)
452
+ assert_false e.for_another_worker?
453
+ end
454
+
455
+ test 'target_worker_id != current worker_id' do
456
+ e = element()
457
+ e.set_target_worker_id(1)
458
+ assert e.for_another_worker?
459
+ end
460
+
461
+ test "doesn't have target_worker_id" do
462
+ e = element()
463
+ assert_false e.for_another_worker?
464
+ end
465
+ end
403
466
  end
@@ -105,8 +105,8 @@ module Fluent::Config
105
105
  test('"\\.t"') { assert_text_parsed_as(".t", '"\\.t"') }
106
106
  test('"\\$t"') { assert_text_parsed_as("$t", '"\\$t"') }
107
107
  test('"\\"') { assert_text_parsed_as("#t", '"\\#t"') }
108
+ test('"\\0"') { assert_text_parsed_as("\0", '"\\0"') }
108
109
  test('"\\z"') { assert_parse_error('"\\z"') } # unknown escaped character
109
- test('"\\0"') { assert_parse_error('"\\0"') } # unknown escaped character
110
110
  test('"\\1"') { assert_parse_error('"\\1"') } # unknown escaped character
111
111
  test('"t') { assert_parse_error('"t') } # non-terminated quoted character
112
112
  test("\"t\nt\"") { assert_text_parsed_as("t\nt", "\"t\nt\"" ) } # multiline string
@@ -13,11 +13,13 @@ module Fluent::Config
13
13
  end
14
14
 
15
15
  class FakeSupervisor
16
+ attr_writer :log_level
17
+
16
18
  def initialize
17
19
  @workers = nil
18
20
  @root_dir = nil
19
21
  @log = FakeLoggerInitializer.new
20
- @log_level = nil
22
+ @log_level = Fluent::Log::LEVEL_INFO
21
23
  @suppress_interval = nil
22
24
  @suppress_config_dump = nil
23
25
  @suppress_repeated_stacktrace = nil
@@ -55,7 +57,7 @@ module Fluent::Config
55
57
  assert_nil(sc.without_source)
56
58
  assert_equal(1, s.instance_variable_get(:@workers))
57
59
  assert_nil(s.instance_variable_get(:@root_dir))
58
- assert_nil(s.instance_variable_get(:@log_level))
60
+ assert_equal(Fluent::Log::LEVEL_INFO, s.instance_variable_get(:@log_level))
59
61
  assert_nil(s.instance_variable_get(:@suppress_repeated_stacktrace))
60
62
  assert_nil(s.instance_variable_get(:@emit_error_log_interval))
61
63
  assert_nil(s.instance_variable_get(:@suppress_config_dump))
@@ -99,7 +101,7 @@ module Fluent::Config
99
101
  sc.apply(s)
100
102
  assert_equal(1, s.instance_variable_get(:@workers))
101
103
  assert_nil(s.instance_variable_get(:@root_dir))
102
- assert_nil(s.instance_variable_get(:@log_level))
104
+ assert_equal(Fluent::Log::LEVEL_INFO, s.instance_variable_get(:@log_level))
103
105
  assert_nil(s.instance_variable_get(:@suppress_repeated_stacktrace))
104
106
  assert_nil(s.instance_variable_get(:@emit_error_log_interval))
105
107
  assert_nil(s.instance_variable_get(:@suppress_config_dump))
@@ -109,16 +111,43 @@ module Fluent::Config
109
111
  assert_nil(s.instance_variable_get(:@dir_permission))
110
112
  end
111
113
 
112
- test 'log_level' do
114
+ data('trace' => Fluent::Log::LEVEL_TRACE,
115
+ 'debug' => Fluent::Log::LEVEL_DEBUG,
116
+ 'info' => Fluent::Log::LEVEL_INFO,
117
+ 'warn' => Fluent::Log::LEVEL_WARN,
118
+ 'error' => Fluent::Log::LEVEL_ERROR,
119
+ 'fatal' => Fluent::Log::LEVEL_FATAL)
120
+ test 'log_level is applied when log_level related command line option is not passed' do |level|
121
+ conf = parse_text(<<-EOS)
122
+ <system>
123
+ log_level #{Fluent::Log::LEVEL_TEXT[level]}
124
+ </system>
125
+ EOS
126
+ s = FakeSupervisor.new
127
+ sc = Fluent::SystemConfig.new(conf)
128
+ sc.attach(s)
129
+ sc.apply(s)
130
+ assert_equal(level, s.instance_variable_get("@log").level)
131
+ end
132
+
133
+ # info is removed because info level can't be specified via command line
134
+ data('trace' => Fluent::Log::LEVEL_TRACE,
135
+ 'debug' => Fluent::Log::LEVEL_DEBUG,
136
+ 'warn' => Fluent::Log::LEVEL_WARN,
137
+ 'error' => Fluent::Log::LEVEL_ERROR,
138
+ 'fatal' => Fluent::Log::LEVEL_FATAL)
139
+ test 'log_level is ignored when log_level related command line option is passed' do |level|
113
140
  conf = parse_text(<<-EOS)
114
141
  <system>
115
- log_level warn
142
+ log_level info
116
143
  </system>
117
144
  EOS
118
145
  s = FakeSupervisor.new
146
+ s.log_level = level
119
147
  sc = Fluent::SystemConfig.new(conf)
148
+ sc.attach(s)
120
149
  sc.apply(s)
121
- assert_equal(Fluent::Log::LEVEL_WARN, s.instance_variable_get("@log").level)
150
+ assert_equal(level, s.instance_variable_get("@log").level)
122
151
  end
123
152
 
124
153
  test 'process global overridable variables' do
@@ -130,6 +159,7 @@ module Fluent::Config
130
159
  EOS
131
160
  s = FakeSupervisor.new
132
161
  sc = Fluent::SystemConfig.new(conf)
162
+ sc.attach(s)
133
163
  sc.apply(s)
134
164
  assert_equal(0655, s.instance_variable_get(:@file_permission))
135
165
  assert_equal(0765, s.instance_variable_get(:@dir_permission))
@@ -69,6 +69,17 @@ class TestConfigTypes < ::Test::Unit::TestCase
69
69
  assert_equal Encoding::UTF_8, Config::STRING_TYPE.call('test', {}).encoding
70
70
  end
71
71
 
72
+ data('latin' => 'Märch',
73
+ 'ascii' => 'ascii',
74
+ 'space' => ' ',
75
+ 'number' => '1',
76
+ 'Hiragana' => 'あいうえお')
77
+ test 'string w/ binary' do |str|
78
+ actual = Config::STRING_TYPE.call(str.b, {})
79
+ assert_equal str, actual
80
+ assert_equal Encoding::UTF_8, actual.encoding
81
+ end
82
+
72
83
  test 'enum' do
73
84
  assert_equal :val, Config::ENUM_TYPE.call('val', {list: [:val, :value, :v]})
74
85
  assert_equal :v, Config::ENUM_TYPE.call('v', {list: [:val, :value, :v]})
@@ -143,6 +154,14 @@ class TestConfigTypes < ::Test::Unit::TestCase
143
154
  assert_raise(RuntimeError.new("unknown type in REFORMAT: foo")){ Config::HASH_TYPE.call("x:1,y:2", {value_type: :foo}) }
144
155
  end
145
156
 
157
+ data('latin' => ['3:Märch', {"3"=>"Märch"}],
158
+ 'ascii' => ['ascii:ascii', {"ascii"=>"ascii"}],
159
+ 'number' => ['number:1', {"number"=>"1"}],
160
+ 'Hiragana' => ['hiragana:あいうえお', {"hiragana"=>"あいうえお"}])
161
+ test 'hash w/ binary' do |(target, expected)|
162
+ assert_equal(expected, Config::HASH_TYPE.call(target.b, { value_type: :string }))
163
+ end
164
+
146
165
  test 'array' do
147
166
  assert_equal(["1","2",1], Config::ARRAY_TYPE.call('["1","2",1]', {}))
148
167
  assert_equal(["1","2","1"], Config::ARRAY_TYPE.call('1,2,1', {}))