fluentd 0.14.13-x64-mingw32 → 0.14.17-x64-mingw32

data/test/command/test_fluentd.rb

@@ -297,7 +297,7 @@ CONF
       assert_log_matches(
         create_cmdline(conf_path),
         "fluentd worker is now running",
-        '[warn]: match for some tags of log events are not defined (to be ignored) tags=["fluent.trace", "fluent.debug", "fluent.info"]',
+        '[warn]: #0 match for some tags of log events are not defined (to be ignored) tags=["fluent.trace", "fluent.debug", "fluent.info"]',
         '[warn]: #0 no patterns matched tag="fluent.info"',
       )
     end
@@ -334,7 +334,7 @@ CONF
       assert_log_matches(
         create_cmdline(conf_path),
         "fluentd worker is now running",
-        '[warn]: match for some tags of log events are not defined (to be ignored) tags=["fluent.info", "fluent.fatal"]',
+        '[warn]: #0 match for some tags of log events are not defined (to be ignored) tags=["fluent.info", "fluent.fatal"]',
         patterns_not_match: ['[warn]: no patterns matched tag="fluent.info"'],
       )
     end
@@ -442,8 +442,7 @@ CONF
 
       assert_fluentd_fails_to_start(
         create_cmdline(conf_path, "-p", File.dirname(plugin_path)),
-        "error_class=SyntaxError",
-        "in_buggy.rb:5: syntax error, unexpected end-of-input, expecting keyword_end",
+        "in_buggy.rb:5: syntax error, unexpected end-of-input, expecting keyword_end (SyntaxError)"
       )
     end
   end
@@ -614,5 +613,255 @@ CONF
         "config error file=\"#{conf_path}\" error_class=Fluent::ConfigError error=\"Plugin 'file' does not support multi workers configuration (Fluent::Plugin::FileBuffer)\"",
       )
     end
+
+    test 'failed to start workers when configured plugins as chidren of MultiOutput do not support multi worker configuration' do
+      script = <<-EOC
+require 'fluent/plugin/output'
+module Fluent::Plugin
+  class SingleOutput < Output
+    Fluent::Plugin.register_output('single', self)
+    def multi_workers_ready?
+      false
+    end
+    def write(chunk)
+    end
+  end
+end
+EOC
+      plugin_path = create_plugin_file('out_single.rb', script)
+
+      conf = <<CONF
+<system>
+  workers 2
+</system>
+<source>
+  @type single
+  @id single
+  @label @dummydata
+</source>
+<label @dummydata>
+  <match dummy>
+    @type copy
+    <store>
+      @type single
+    </store>
+    <store>
+      @type single
+    </store>
+  </match>
+</label>
+CONF
+      conf_path = create_conf_file('workers_invalid3.conf', conf)
+      assert_fluentd_fails_to_start(
+        create_cmdline(conf_path, "-p", File.dirname(plugin_path)),
+        "Plugin 'single' does not support multi workers configuration (Fluent::Plugin::SingleOutput)",
+      )
+    end
+
+    test 'success to start a worker with worker specific configuration' do
+      conf = <<CONF
+<system>
+  workers 2
+  root_dir #{@root_path}
+</system>
+<source>
+  @type dummy
+  @id dummy
+  @label @dummydata
+  tag dummy
+  dummy {"message": "yay!"}
+</source>
+<worker 1>
+  <source>
+    @type dummy
+    @id dummy_in_worker
+    @label @dummydata
+    tag dummy
+    dummy {"message": "yay!"}
+  </source>
+</worker>
+<label @dummydata>
+  <match dummy>
+    @type null
+    @id   blackhole
+  </match>
+</label>
+CONF
+      conf_path = create_conf_file('worker_section0.conf', conf)
+      assert Dir.exist?(@root_path)
+
+      assert_log_matches(
+        create_cmdline(conf_path),
+        "#0 fluentd worker is now running worker=0",
+        "#1 fluentd worker is now running worker=1",
+        /(?!#\d) adding source type="dummy"/,
+        '#1 adding source type="dummy"'
+      )
+    end
+
+    test 'success to start workers when configured plugins only for specific worker do not support multi worker configuration' do
+      script =  <<-EOC
+require 'fluent/plugin/input'
+module Fluent::Plugin
+  class SingleInput < Input
+    Fluent::Plugin.register_input('single', self)
+    def multi_workers_ready?
+      false
+    end
+  end
+end
+EOC
+      plugin_path = create_plugin_file('in_single.rb', script)
+
+      conf = <<CONF
+<system>
+  workers 2
+</system>
+<worker 1>
+  <source>
+    @type single
+    @id single
+    @label @dummydata
+  </source>
+</worker>
+<label @dummydata>
+  <match dummy>
+    @type null
+    @id   blackhole
+  </match>
+</label>
+CONF
+      conf_path = create_conf_file('worker_section1.conf', conf)
+      assert Dir.exist?(@root_path)
+
+      assert_log_matches(
+        create_cmdline(conf_path, "-p", File.dirname(plugin_path)),
+        "#0 fluentd worker is now running worker=0",
+        "#1 fluentd worker is now running worker=1",
+        '#1 adding source type="single"'
+      )
+    end
+
+    test 'success to start workers when file buffer is configured in non-workers way only for specific worker' do
+      conf = <<CONF
+<system>
+  workers 2
+</system>
+<source>
+  @type dummy
+  @id dummy
+  tag dummy
+  dummy {"message": "yay!"}
+</source>
+<worker 1>
+  <match dummy>
+    @type null
+    @id   blackhole
+    <buffer>
+      @type file
+      path #{File.join(@root_path, "buf", "file.*.log")}
+    </buffer>
+  </match>
+</worker>
+CONF
+      conf_path = create_conf_file('worker_section2.conf', conf)
+      assert_log_matches(
+        create_cmdline(conf_path),
+        "#0 fluentd worker is now running worker=0",
+        "#1 fluentd worker is now running worker=1",
+        '#1 adding match pattern="dummy" type="null"'
+      )
+    end
+
+    test 'success to start workers when configured plugins as a chidren of MultiOutput only for specific worker do not support multi worker configuration' do
+      script = <<-EOC
+require 'fluent/plugin/output'
+module Fluent::Plugin
+  class SingleOutput < Output
+    Fluent::Plugin.register_output('single', self)
+    def multi_workers_ready?
+      false
+    end
+    def write(chunk)
+    end
+  end
+end
+EOC
+      plugin_path = create_plugin_file('out_single.rb', script)
+
+      conf = <<CONF
+<system>
+  workers 2
+</system>
+<source>
+  @type dummy
+  @id dummy
+  tag dummy
+  dummy {"message": "yay!"}
+</source>
+<worker 1>
+  <match dummy>
+    @type copy
+    <store>
+      @type single
+    </store>
+    <store>
+      @type single
+    </store>
+  </match>
+</worker>
+CONF
+      conf_path = create_conf_file('worker_section3.conf', conf)
+      assert_log_matches(
+        create_cmdline(conf_path, "-p", File.dirname(plugin_path)),
+        "#0 fluentd worker is now running worker=0",
+        "#1 fluentd worker is now running worker=1",
+        '#1 adding match pattern="dummy" type="copy"'
+      )
+    end
+  end
+
+  sub_test_case 'config dump' do
+    test 'all secret parameters in worker section is sealed' do
+      script =  <<-EOC
+require 'fluent/plugin/input'
+module Fluent::Plugin
+  class FakeInput < Input
+    Fluent::Plugin.register_input('fake', self)
+    config_param :secret, :string, secret: true
+  end
+end
+EOC
+      plugin_path = create_plugin_file('in_fake.rb', script)
+
+      conf = <<CONF
+<system>
+  workers 2
+</system>
+<worker 0>
+  <source>
+    @type fake
+    secret secret0
+  </source>
+  <match>
+    @type null
+  </match>
+</worker>
+<worker 1>
+  <source>
+    @type fake
+    secret secret1
+  </source>
+  <match>
+    @type null
+  </match>
+</worker>
+CONF
+      conf_path = create_conf_file('secret_in_worker.conf', conf)
+      assert File.exist?(conf_path)
+
+      assert_log_matches(create_cmdline(conf_path, "-p", File.dirname(plugin_path)),
+                         "secret xxxxxx", patterns_not_match: ["secret secret0", "secret secret1"])
+    end
   end
 end

data/test/config/test_element.rb

@@ -400,4 +400,67 @@ CONF
       end
     end
   end
+
+  sub_test_case '#set_target_worker' do
+    test 'set target_worker_id recursively' do
+      e = element('label', '@mytest', {}, [ element('filter', '**'), element('match', '**', {}, [ element('store'), element('store') ]) ])
+      e.set_target_worker_id(1)
+      assert_equal 1, e.target_worker_id
+      assert_equal 1, e.elements[0].target_worker_id
+      assert_equal 1, e.elements[1].target_worker_id
+      assert_equal 1, e.elements[1].elements[0].target_worker_id
+      assert_equal 1, e.elements[1].elements[1].target_worker_id
+    end
+  end
+
+  sub_test_case '#for_every_workers?' do
+    test 'has target_worker_id' do
+      e = element()
+      e.set_target_worker_id(1)
+      assert_false e.for_every_workers?
+    end
+
+    test "doesn't have target_worker_id" do
+      e = element()
+      assert e.for_every_workers?
+    end
+  end
+
+  sub_test_case '#for_this_workers?' do
+    test 'target_worker_id == current worker_id' do
+      e = element()
+      e.set_target_worker_id(0)
+      assert e.for_this_worker?
+    end
+
+    test 'target_worker_id != current worker_id' do
+      e = element()
+      e.set_target_worker_id(1)
+      assert_false e.for_this_worker?
+    end
+
+    test "doesn't have target_worker_id" do
+      e = element()
+      assert_false e.for_this_worker?
+    end
+  end
+
+  sub_test_case '#for_another_worker?' do
+    test 'target_worker_id == current worker_id' do
+      e = element()
+      e.set_target_worker_id(0)
+      assert_false e.for_another_worker?
+    end
+
+    test 'target_worker_id != current worker_id' do
+      e = element()
+      e.set_target_worker_id(1)
+      assert e.for_another_worker?
+    end
+
+    test "doesn't have target_worker_id" do
+      e = element()
+      assert_false e.for_another_worker?
+    end
+  end
 end

data/test/config/test_literal_parser.rb

@@ -105,8 +105,8 @@ module Fluent::Config
       test('"\\.t"') { assert_text_parsed_as(".t", '"\\.t"') }
       test('"\\$t"') { assert_text_parsed_as("$t", '"\\$t"') }
       test('"\\"') { assert_text_parsed_as("#t", '"\\#t"') }
+      test('"\\0"') { assert_text_parsed_as("\0", '"\\0"') }
       test('"\\z"') { assert_parse_error('"\\z"') }  # unknown escaped character
-      test('"\\0"') { assert_parse_error('"\\0"') }  # unknown escaped character
       test('"\\1"') { assert_parse_error('"\\1"') }  # unknown escaped character
       test('"t') { assert_parse_error('"t') }  # non-terminated quoted character
       test("\"t\nt\"") { assert_text_parsed_as("t\nt", "\"t\nt\"" ) } # multiline string

data/test/config/test_system_config.rb

@@ -13,11 +13,13 @@ module Fluent::Config
   end
 
   class FakeSupervisor
+    attr_writer :log_level
+
     def initialize
       @workers = nil
       @root_dir = nil
       @log = FakeLoggerInitializer.new
-      @log_level = nil
+      @log_level = Fluent::Log::LEVEL_INFO
       @suppress_interval = nil
       @suppress_config_dump = nil
       @suppress_repeated_stacktrace = nil
@@ -55,7 +57,7 @@ module Fluent::Config
       assert_nil(sc.without_source)
       assert_equal(1, s.instance_variable_get(:@workers))
       assert_nil(s.instance_variable_get(:@root_dir))
-      assert_nil(s.instance_variable_get(:@log_level))
+      assert_equal(Fluent::Log::LEVEL_INFO, s.instance_variable_get(:@log_level))
       assert_nil(s.instance_variable_get(:@suppress_repeated_stacktrace))
       assert_nil(s.instance_variable_get(:@emit_error_log_interval))
       assert_nil(s.instance_variable_get(:@suppress_config_dump))
@@ -99,7 +101,7 @@ module Fluent::Config
       sc.apply(s)
       assert_equal(1, s.instance_variable_get(:@workers))
       assert_nil(s.instance_variable_get(:@root_dir))
-      assert_nil(s.instance_variable_get(:@log_level))
+      assert_equal(Fluent::Log::LEVEL_INFO, s.instance_variable_get(:@log_level))
       assert_nil(s.instance_variable_get(:@suppress_repeated_stacktrace))
       assert_nil(s.instance_variable_get(:@emit_error_log_interval))
       assert_nil(s.instance_variable_get(:@suppress_config_dump))
@@ -109,16 +111,43 @@ module Fluent::Config
       assert_nil(s.instance_variable_get(:@dir_permission))
     end
 
-    test 'log_level' do
+    data('trace' => Fluent::Log::LEVEL_TRACE,
+         'debug' => Fluent::Log::LEVEL_DEBUG,
+         'info' => Fluent::Log::LEVEL_INFO,
+         'warn' => Fluent::Log::LEVEL_WARN,
+         'error' => Fluent::Log::LEVEL_ERROR,
+         'fatal' => Fluent::Log::LEVEL_FATAL)
+    test 'log_level is applied when log_level related command line option is not passed' do |level|
+      conf = parse_text(<<-EOS)
+        <system>
+          log_level #{Fluent::Log::LEVEL_TEXT[level]}
+        </system>
+      EOS
+      s = FakeSupervisor.new
+      sc = Fluent::SystemConfig.new(conf)
+      sc.attach(s)
+      sc.apply(s)
+      assert_equal(level, s.instance_variable_get("@log").level)
+    end
+
+    # info is removed because info level can't be specified via command line
+    data('trace' => Fluent::Log::LEVEL_TRACE,
+         'debug' => Fluent::Log::LEVEL_DEBUG,
+         'warn' => Fluent::Log::LEVEL_WARN,
+         'error' => Fluent::Log::LEVEL_ERROR,
+         'fatal' => Fluent::Log::LEVEL_FATAL)
+    test 'log_level is ignored when log_level related command line option is passed' do |level|
       conf = parse_text(<<-EOS)
         <system>
-          log_level warn
+          log_level info
         </system>
       EOS
       s = FakeSupervisor.new
+      s.log_level = level
       sc = Fluent::SystemConfig.new(conf)
+      sc.attach(s)
       sc.apply(s)
-      assert_equal(Fluent::Log::LEVEL_WARN, s.instance_variable_get("@log").level)
+      assert_equal(level, s.instance_variable_get("@log").level)
     end
 
     test 'process global overridable variables' do
@@ -130,6 +159,7 @@ module Fluent::Config
       EOS
       s = FakeSupervisor.new
       sc = Fluent::SystemConfig.new(conf)
+      sc.attach(s)
       sc.apply(s)
       assert_equal(0655, s.instance_variable_get(:@file_permission))
       assert_equal(0765, s.instance_variable_get(:@dir_permission))

data/test/config/test_types.rb

@@ -69,6 +69,17 @@ class TestConfigTypes < ::Test::Unit::TestCase
       assert_equal Encoding::UTF_8, Config::STRING_TYPE.call('test', {}).encoding
     end
 
+    data('latin' => 'Märch',
+         'ascii' => 'ascii',
+         'space' => '     ',
+         'number' => '1',
+         'Hiragana' => 'あいうえお')
+    test 'string w/ binary' do |str|
+      actual = Config::STRING_TYPE.call(str.b, {})
+      assert_equal str, actual
+      assert_equal Encoding::UTF_8, actual.encoding
+    end
+
     test 'enum' do
       assert_equal :val, Config::ENUM_TYPE.call('val', {list: [:val, :value, :v]})
       assert_equal :v, Config::ENUM_TYPE.call('v', {list: [:val, :value, :v]})
@@ -143,6 +154,14 @@ class TestConfigTypes < ::Test::Unit::TestCase
       assert_raise(RuntimeError.new("unknown type in REFORMAT: foo")){ Config::HASH_TYPE.call("x:1,y:2", {value_type: :foo}) }
     end
 
+    data('latin' => ['3:Märch', {"3"=>"Märch"}],
+         'ascii' => ['ascii:ascii', {"ascii"=>"ascii"}],
+         'number' => ['number:1', {"number"=>"1"}],
+         'Hiragana' => ['hiragana:あいうえお', {"hiragana"=>"あいうえお"}])
+    test 'hash w/ binary' do |(target, expected)|
+      assert_equal(expected, Config::HASH_TYPE.call(target.b, { value_type: :string }))
+    end
+
     test 'array' do
       assert_equal(["1","2",1], Config::ARRAY_TYPE.call('["1","2",1]', {}))
       assert_equal(["1","2","1"], Config::ARRAY_TYPE.call('1,2,1', {}))