fluentd 1.8.1-x86-mingw32 → 1.9.0-x86-mingw32

data/test/plugin_helper/test_server.rb

@@ -751,6 +751,19 @@ class ServerPluginHelperTest < Test::Unit::TestCase
       assert_equal 1, errors.size
       assert_equal "BUG: this event is disabled for udp: close", errors.first.message
     end
+
+    test 'can bind IPv4 / IPv6 together' do
+      omit "IPv6 unavailable here" unless ipv6_enabled?
+
+      assert_nothing_raised do
+        @d.server_create_udp(:s_ipv4_udp, PORT, bind: '0.0.0.0', shared: false, max_bytes: 128) do |data, sock|
+          # ...
+        end
+        @d.server_create_udp(:s_ipv6_udp, PORT, bind: '::', shared: false, max_bytes: 128) do |data, sock|
+          # ...
+        end
+      end
+    end
   end
 
   module CertUtil

data/test/plugin_helper/test_timer.rb

@@ -34,12 +34,12 @@ class TimerTest < Test::Unit::TestCase
       counter += 1
     end
 
-    sleep 5
+    sleep 2
 
     d1.stop
     assert !d1.timer_running?
 
-    assert{ counter >= 3 && counter <= 6 }
+    assert{ counter >= 1 && counter <= 2 }
 
     d1.shutdown; d1.close; d1.terminate
   end
@@ -52,19 +52,18 @@ class TimerTest < Test::Unit::TestCase
     counter1 = 0
     counter2 = 0
 
-    d1.timer_execute(:t1, 1) do
+    d1.timer_execute(:t1, 0.2) do
       counter1 += 1
     end
-    d1.timer_execute(:t2, 2) do
+    d1.timer_execute(:t2, 0.2) do
       counter2 += 1
     end
 
-    sleep 6
-
+    sleep 1
     d1.stop
 
-    assert{ counter1 >= 4 && counter1 <= 7 }
-    assert{ counter2 >= 2 && counter2 <= 4 }
+    assert{ counter1 >= 4 && counter1 <= 5 }
+    assert{ counter2 >= 4 && counter2 <= 5 }
 
     d1.shutdown; d1.close; d1.terminate
   end
@@ -77,19 +76,18 @@ class TimerTest < Test::Unit::TestCase
     counter1 = 0
     counter2 = 0
 
-    d1.timer_execute(:t1, 1) do
+    d1.timer_execute(:t1, 0.2) do
       counter1 += 1
     end
-    d1.timer_execute(:t2, 1) do
+    d1.timer_execute(:t2, 0.2) do
       raise "abort!!!!!!" if counter2 > 1
       counter2 += 1
     end
 
-    sleep 5
-
+    sleep 1
     d1.stop
 
-    assert{ counter1 >= 3 && counter1 <= 6 }
+    assert{ counter1 >= 4 && counter1 <= 5 }
     assert{ counter2 == 2 }
     msg = "Unexpected error raised. Stopping the timer. title=:t2"
     assert{ d1.log.out.logs.any?{|line| line.include?("[error]:") && line.include?(msg) && line.include?("abort!!!!!!") } }

data/test/test_config.rb

@@ -151,16 +151,16 @@ class ConfigTest < Test::Unit::TestCase
     assert_equal before_size, match_conf.unused.size
   end
 
-  def write_config(path, data)
+  def write_config(path, data, encoding: 'utf-8')
     FileUtils.mkdir_p(File.dirname(path))
-    File.open(path, "w") {|f| f.write data }
+    File.open(path, "w:#{encoding}:utf-8") {|f| f.write data }
   end
 
   def test_inline
     prepare_config
     opts = {
-        :config_path => "#{TMP_DIR}/config_test_1.conf",
-        :inline_config => "<source>\n  type http\n  port 2222\n </source>"
+      :config_path => "#{TMP_DIR}/config_test_1.conf",
+      :inline_config => "<source>\n  type http\n  port 2222\n </source>"
     }
     assert_nothing_raised do
       Fluent::Supervisor.new(opts)
@@ -175,5 +175,27 @@ class ConfigTest < Test::Unit::TestCase
     logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)
     $log = Fluent::Log.new(logger)
   end
-end
 
+  sub_test_case '.build' do
+    test 'read config' do
+      write_config("#{TMP_DIR}/build/config_build.conf", 'key value')
+      c = Fluent::Config.build(config_path: "#{TMP_DIR}/build/config_build.conf")
+      assert_equal('value', c['key'])
+    end
+
+    test 'read config with encoding' do
+      write_config("#{TMP_DIR}/build/config_build2.conf", "#てすと\nkey value", encoding: 'shift_jis')
+
+      c = Fluent::Config.build(config_path: "#{TMP_DIR}/build/config_build2.conf", encoding: 'shift_jis')
+      assert_equal('value', c['key'])
+    end
+
+    test 'read config with additional_config' do
+      write_config("#{TMP_DIR}/build/config_build2.conf", "key value")
+
+      c = Fluent::Config.build(config_path: "#{TMP_DIR}/build/config_build2.conf", additional_config: 'key2 value2')
+      assert_equal('value', c['key'])
+      assert_equal('value2', c['key2'])
+    end
+  end
+end

data/test/test_engine.rb

@@ -0,0 +1,203 @@
+require_relative 'helper'
+require 'fluent/engine'
+require 'fluent/config'
+require 'fluent/input'
+require 'fluent/system_config'
+
+class EngineTest < ::Test::Unit::TestCase
+  class DummyEngineTestOutput < Fluent::Plugin::Output
+    Fluent::Plugin.register_output('dummy_engine_test', self)
+    def write(chunk); end
+  end
+
+  class DummyEngineTest2Output < Fluent::Plugin::Output
+    Fluent::Plugin.register_output('dummy_engine_test2', self)
+    def write(chunk); end
+  end
+
+  class DummyEngineTestInput < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('dummy_engine_test', self)
+    def multi_workers_ready?; true; end
+  end
+
+  class DummyEngineTest2Input < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('dummy_engine_test2', self)
+    def multi_workers_ready?; true; end
+  end
+
+  class DummyEngineClassVarTestInput < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('dummy_engine_class_var_test', self)
+    @@test = nil
+    def multi_workers_ready?; true; end
+  end
+
+  sub_test_case '#reload_config' do
+    test 'reload new configuration' do
+      conf_data = <<-CONF
+        <source>
+          @type dummy_engine_test
+        </source>
+        <match>
+          @type dummy_engine_test
+        </match>
+      CONF
+
+      conf = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
+      system_config = Fluent::SystemConfig.create(conf)
+
+      engine = Fluent::EngineClass.new
+      engine.init(system_config)
+      engine.configure(conf)
+
+      assert_kind_of DummyEngineTestInput, engine.root_agent.inputs[0]
+      assert_kind_of DummyEngineTestOutput, engine.root_agent.outputs[0]
+
+      new_conf_data = <<-CONF
+        <source>
+          @type dummy_engine_test2
+        </source>
+        <match>
+          @type dummy_engine_test2
+        </match>
+      CONF
+
+      new_conf = Fluent::Config.parse(new_conf_data, '(test)', '(test_dir)', true)
+
+      agent = Fluent::RootAgent.new(log: $log, system_config: system_config)
+      stub(Fluent::RootAgent).new do
+        stub(agent).start.once
+        agent
+      end
+
+      engine.reload_config(new_conf)
+
+      assert_kind_of DummyEngineTest2Input, engine.root_agent.inputs[0]
+      assert_kind_of DummyEngineTest2Output, engine.root_agent.outputs[0]
+    end
+
+    test "doesn't start RootAgent when supervisor is true" do
+      conf_data = <<-CONF
+        <source>
+          @type dummy_engine_test
+        </source>
+        <match>
+          @type dummy_engine_test
+        </match>
+      CONF
+
+      conf = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
+      system_config = Fluent::SystemConfig.create(conf)
+
+      engine = Fluent::EngineClass.new
+      engine.init(system_config)
+      engine.configure(conf)
+
+      assert_kind_of DummyEngineTestInput, engine.root_agent.inputs[0]
+      assert_kind_of DummyEngineTestOutput, engine.root_agent.outputs[0]
+
+      new_conf_data = <<-CONF
+        <source>
+          @type dummy_engine_test2
+        </source>
+        <match>
+          @type dummy_engine_test2
+        </match>
+      CONF
+
+      new_conf = Fluent::Config.parse(new_conf_data, '(test)', '(test_dir)', true)
+
+      agent = Fluent::RootAgent.new(log: $log, system_config: system_config)
+      stub(Fluent::RootAgent).new do
+        stub(agent).start.never
+        agent
+      end
+
+      engine.reload_config(new_conf, supervisor: true)
+
+      assert_kind_of DummyEngineTest2Input, engine.root_agent.inputs[0]
+      assert_kind_of DummyEngineTest2Output, engine.root_agent.outputs[0]
+    end
+
+    test 'raise an error when conf is invalid' do
+      conf_data = <<-CONF
+        <source>
+          @type dummy_engine_test
+        </source>
+        <match>
+          @type dummy_engine_test
+        </match>
+      CONF
+
+      conf = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
+      system_config = Fluent::SystemConfig.create(conf)
+
+      engine = Fluent::EngineClass.new
+      engine.init(system_config)
+      engine.configure(conf)
+
+      assert_kind_of DummyEngineTestInput, engine.root_agent.inputs[0]
+      assert_kind_of DummyEngineTestOutput, engine.root_agent.outputs[0]
+
+      new_conf_data = <<-CONF
+        <source>
+          @type
+        </source>
+      CONF
+
+      new_conf = Fluent::Config.parse(new_conf_data, '(test)', '(test_dir)', true)
+
+      agent = Fluent::RootAgent.new(log: $log, system_config: system_config)
+      stub(Fluent::RootAgent).new do
+        stub(agent).start.never
+        agent
+      end
+
+      assert_raise(Fluent::ConfigError.new("Missing '@type' parameter on <source> directive")) do
+        engine.reload_config(new_conf)
+      end
+
+      assert_kind_of DummyEngineTestInput, engine.root_agent.inputs[0]
+      assert_kind_of DummyEngineTestOutput, engine.root_agent.outputs[0]
+    end
+
+    test 'raise an error when unreloadable exists' do
+      conf_data = <<-CONF
+        <source>
+          @type dummy_engine_test
+        </source>
+        <match>
+          @type dummy_engine_test
+        </match>
+      CONF
+
+      conf = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
+      system_config = Fluent::SystemConfig.create(conf)
+
+      engine = Fluent::EngineClass.new
+      engine.init(system_config)
+      engine.configure(conf)
+
+      assert_kind_of DummyEngineTestInput, engine.root_agent.inputs[0]
+      assert_kind_of DummyEngineTestOutput, engine.root_agent.outputs[0]
+
+      conf_data = <<-CONF
+        <source>
+          @type dummy_engine_class_var_test
+        </source>
+        <match>
+          @type dummy_engine_test
+        </match>
+      CONF
+
+      new_conf = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
+
+      e = assert_raise(Fluent::ConfigError) do
+        engine.reload_config(new_conf)
+      end
+      assert e.message.match?('Unreloadable plugin plugin: dummy_engine_class_var_test')
+
+      assert_kind_of DummyEngineTestInput, engine.root_agent.inputs[0]
+      assert_kind_of DummyEngineTestOutput, engine.root_agent.outputs[0]
+    end
+  end
+end

data/test/test_output.rb

@@ -116,7 +116,7 @@ module FluentOutputTest
         end
       end
 
-      mock(d.instance.log).warn("secondary type should be same with primary one",
+      mock(d.instance.log).warn("Use different plugin for secondary. Check the plugin works with primary like secondary_file",
                                 { primary: d.instance.class.to_s, secondary: "Fluent::Plugin::Test2Output" })
       d.configure(CONFIG + %[
         <secondary>
@@ -132,7 +132,7 @@ module FluentOutputTest
       # ObjectBufferedOutput doesn't implement `custom_filter`
       d = Fluent::Test::BufferedOutputTestDriver.new(Fluent::ObjectBufferedOutput)
 
-      mock(d.instance.log).warn("secondary type should be same with primary one",
+      mock(d.instance.log).warn("Use different plugin for secondary. Check the plugin works with primary like secondary_file",
                                 { primary: d.instance.class.to_s, secondary: "Fluent::Plugin::Test2Output" }).never
       d.configure(CONFIG + %[
         <secondary>

data/test/test_static_config_analysis.rb

@@ -0,0 +1,177 @@
+require_relative 'helper'
+
+require 'fluent/config'
+require 'fluent/static_config_analysis'
+require 'fluent/plugin/out_forward'
+require 'fluent/plugin/out_stdout'
+require 'fluent/plugin/out_exec'
+require 'fluent/plugin/in_forward'
+require 'fluent/plugin/in_dummy'
+require 'fluent/plugin/filter_grep'
+require 'fluent/plugin/filter_stdout'
+require 'fluent/plugin/filter_parser'
+
+class StaticConfigAnalysisTest < ::Test::Unit::TestCase
+  sub_test_case '.call' do
+    test 'returns outputs, inputs and filters' do
+      conf_data = <<-CONF
+<source>
+  @type forward
+</source>
+<filter>
+  @type grep
+</filter>
+<match>
+  @type forward
+</match>
+      CONF
+
+      c = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
+      ret = Fluent::StaticConfigAnalysis.call(c)
+      assert_equal 1, ret.outputs.size
+      assert_kind_of Fluent::Plugin::ForwardOutput, ret.outputs[0].plugin
+      assert_equal 1, ret.inputs.size
+      assert_kind_of Fluent::Plugin::ForwardInput, ret.inputs[0].plugin
+      assert_equal 1, ret.filters.size
+      assert_kind_of Fluent::Plugin::GrepFilter, ret.filters[0].plugin
+      assert_empty ret.labels
+
+      assert_equal [Fluent::Plugin::ForwardOutput, Fluent::Plugin::ForwardInput, Fluent::Plugin::GrepFilter], ret.all_plugins.map(&:class)
+    end
+
+    test 'returns wrapped element with worker and label section' do
+      conf_data = <<-CONF
+<source>
+  @type forward
+</source>
+<filter>
+  @type grep
+</filter>
+<match>
+  @type forward
+</match>
+<worker 0>
+  <source>
+    @type dummy
+  </source>
+  <filter>
+    @type parser
+  </filter>
+  <match>
+    @type exec
+  </match>
+</worker>
+<label @test>
+  <filter>
+    @type stdout
+  </filter>
+  <match>
+    @type stdout
+  </match>
+</label>
+      CONF
+
+      c = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
+      ret = Fluent::StaticConfigAnalysis.call(c)
+      assert_equal [Fluent::Plugin::ExecOutput, Fluent::Plugin::StdoutOutput, Fluent::Plugin::ForwardOutput], ret.outputs.map(&:plugin).map(&:class)
+      assert_equal [Fluent::Plugin::DummyInput, Fluent::Plugin::ForwardInput], ret.inputs.map(&:plugin).map(&:class)
+      assert_equal [Fluent::Plugin::ParserFilter, Fluent::Plugin::StdoutFilter, Fluent::Plugin::GrepFilter], ret.filters.map(&:plugin).map(&:class)
+      assert_equal 1, ret.labels.size
+      assert_equal '@test', ret.labels[0].name
+    end
+
+    sub_test_case 'raises config error' do
+      data(
+        'empty' => ['', 'Missing worker id on <worker> directive'],
+        'invalid number' => ['a', 'worker id should be integer: a'],
+        'worker id is negative' => ['-1', 'worker id should be integer: -1'],
+        'min worker id is less than 0' => ['-1-1', 'worker id should be integer: -1-1'],
+        'max worker id is less than 0' => ['1--1', 'worker id -1 specified by <worker> directive is not allowed. Available worker id is between 0 and 1'],
+        'min worker id is greater than workers' => ['0-2', 'worker id 2 specified by <worker> directive is not allowed. Available worker id is between 0 and 1'],
+        'max worker is less than min worker' => ['1-0', "greater first_worker_id<1> than last_worker_id<0> specified by <worker> directive is not allowed. Available multi worker assign syntax is <smaller_worker_id>-<greater_worker_id>"],
+      )
+      test 'when worker number is invalid' do |v|
+        val, msg = v
+        conf_data = <<-CONF
+<worker #{val}>
+</worker>
+CONF
+
+        c = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
+        assert_raise(Fluent::ConfigError.new(msg)) do
+          Fluent::StaticConfigAnalysis.call(c, workers: 2)
+        end
+      end
+
+      test 'when worker number is duplicated' do
+        conf_data = <<-CONF
+<worker 0-1>
+</worker>
+<worker 0-1>
+</worker>
+CONF
+
+        c = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
+        assert_raise(Fluent::ConfigError.new("specified worker_id<0> collisions is detected on <worker> directive. Available worker id(s): []")) do
+          Fluent::StaticConfigAnalysis.call(c, workers: 2)
+        end
+      end
+
+      test 'duplicated label exits' do
+        conf_data = <<-CONF
+<label @dup>
+</label>
+<label @dup>
+</label>
+CONF
+
+        c = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
+        assert_raise(Fluent::ConfigError.new('Section <label @dup> appears twice')) do
+          Fluent::StaticConfigAnalysis.call(c, workers: 2)
+        end
+      end
+
+      test 'empty label' do
+        conf_data = <<-CONF
+<label>
+</label>
+CONF
+
+        c = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
+        assert_raise(Fluent::ConfigError.new('Missing symbol argument on <label> directive')) do
+          Fluent::StaticConfigAnalysis.call(c, workers: 2)
+        end
+      end
+
+      data(
+        'in filter' => 'filter',
+        'in source' => 'source',
+        'in match' => 'match',
+      )
+      test 'when @type is missing' do |name|
+        conf_data = <<-CONF
+<#{name}>
+  @type
+</#{name}>
+CONF
+        c = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
+        assert_raise(Fluent::ConfigError.new("Missing '@type' parameter on <#{name}> directive")) do
+          Fluent::StaticConfigAnalysis.call(c)
+        end
+      end
+
+      test 'when worker has worker section' do
+        conf_data = <<-CONF
+<worker 0>
+  <worker 0>
+  </worker>
+</worker>
+CONF
+        c = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
+        assert_raise(Fluent::ConfigError.new("<worker> section cannot have <worker> directive")) do
+          Fluent::StaticConfigAnalysis.call(c)
+        end
+      end
+    end
+  end
+end