fluentd 1.14.4-x64-mingw-ucrt

data/test/test_config.rb

@@ -0,0 +1,202 @@
+require_relative 'helper'
+require 'fluent/config'
+require 'fluent/config/parser'
+require 'fluent/supervisor'
+require 'fluent/load'
+require 'fileutils'
+
+class ConfigTest < Test::Unit::TestCase
+  include Fluent
+
+  TMP_DIR = File.dirname(__FILE__) + "/tmp/config#{ENV['TEST_ENV_NUMBER']}"
+
+  def read_config(path)
+    path = File.expand_path(path)
+    File.open(path) { |io|
+      Fluent::Config::Parser.parse(io, File.basename(path), File.dirname(path))
+    }
+  end
+
+  def prepare_config
+    write_config "#{TMP_DIR}/config_test_1.conf", %[
+      k1 root_config
+      include dir/config_test_2.conf  #
+      include #{TMP_DIR}/config_test_4.conf
+      include file://#{TMP_DIR}/config_test_5.conf
+      <include config.d/*.conf />
+    ]
+    write_config "#{TMP_DIR}/dir/config_test_2.conf", %[
+      k2 relative_path_include
+      include ../config_test_3.conf
+    ]
+    write_config "#{TMP_DIR}/config_test_3.conf", %[
+      k3 relative_include_in_included_file
+    ]
+    write_config "#{TMP_DIR}/config_test_4.conf", %[
+      k4 absolute_path_include
+    ]
+    write_config "#{TMP_DIR}/config_test_5.conf", %[
+      k5 uri_include
+    ]
+    write_config "#{TMP_DIR}/config.d/config_test_6.conf", %[
+      k6 wildcard_include_1
+      <elem1 name>
+        include normal_parameter
+      </elem1>
+    ]
+    write_config "#{TMP_DIR}/config.d/config_test_7.conf", %[
+      k7 wildcard_include_2
+    ]
+    write_config "#{TMP_DIR}/config.d/config_test_8.conf", %[
+      <elem2 name>
+        <include ../dir/config_test_9.conf />
+      </elem2>
+    ]
+    write_config "#{TMP_DIR}/dir/config_test_9.conf", %[
+      k9 embedded
+      <elem3 name>
+        nested nested_value
+        include hoge
+      </elem3>
+    ]
+    write_config "#{TMP_DIR}/config.d/00_config_test_8.conf", %[
+      k8 wildcard_include_3
+      <elem4 name>
+        include normal_parameter
+      </elem4>
+    ]
+
+  end
+
+  def test_include
+    prepare_config
+    c = read_config("#{TMP_DIR}/config_test_1.conf")
+    assert_equal 'root_config', c['k1']
+    assert_equal 'relative_path_include', c['k2']
+    assert_equal 'relative_include_in_included_file', c['k3']
+    assert_equal 'absolute_path_include', c['k4']
+    assert_equal 'uri_include', c['k5']
+    assert_equal 'wildcard_include_1', c['k6']
+    assert_equal 'wildcard_include_2', c['k7']
+    assert_equal 'wildcard_include_3', c['k8']
+    assert_equal [
+      'k1',
+      'k2',
+      'k3',
+      'k4',
+      'k5',
+      'k8', # Because of the file name this comes first.
+      'k6',
+      'k7',
+    ], c.keys
+
+    elem1 = c.elements.find { |e| e.name == 'elem1' }
+    assert_not_nil elem1
+    assert_equal 'name', elem1.arg
+    assert_equal 'normal_parameter', elem1['include']
+
+    elem2 = c.elements.find { |e| e.name == 'elem2' }
+    assert_not_nil elem2
+    assert_equal 'name', elem2.arg
+    assert_equal 'embedded', elem2['k9']
+    assert !elem2.has_key?('include')
+
+    elem3 = elem2.elements.find { |e| e.name == 'elem3' }
+    assert_not_nil elem3
+    assert_equal 'nested_value', elem3['nested']
+    assert_equal 'hoge', elem3['include']
+  end
+
+  def test_check_not_fetchd
+    write_config "#{TMP_DIR}/config_test_not_fetched.conf", %[
+      <match dummy>
+       type          rewrite
+       add_prefix    filtered
+       <rule>
+         key     path
+         pattern ^[A-Z]+
+         replace
+       </rule>
+     </match>
+    ]
+    root_conf  = read_config("#{TMP_DIR}/config_test_not_fetched.conf")
+    match_conf = root_conf.elements.first
+    rule_conf  = match_conf.elements.first
+
+    not_fetched = []; root_conf.check_not_fetched {|key, e| not_fetched << key }
+    assert_equal %w[type add_prefix key pattern replace], not_fetched
+
+    not_fetched = []; match_conf.check_not_fetched {|key, e| not_fetched << key }
+    assert_equal %w[type add_prefix key pattern replace], not_fetched
+
+    not_fetched = []; rule_conf.check_not_fetched {|key, e| not_fetched << key }
+    assert_equal %w[key pattern replace], not_fetched
+
+    # accessing should delete
+    match_conf['type']
+    rule_conf['key']
+
+    not_fetched = []; root_conf.check_not_fetched {|key, e| not_fetched << key }
+    assert_equal %w[add_prefix pattern replace], not_fetched
+
+    not_fetched = []; match_conf.check_not_fetched {|key, e| not_fetched << key }
+    assert_equal %w[add_prefix pattern replace], not_fetched
+
+    not_fetched = []; rule_conf.check_not_fetched {|key, e| not_fetched << key }
+    assert_equal %w[pattern replace], not_fetched
+
+    # repeatedly accessing should not grow memory usage
+    before_size = match_conf.unused.size
+    10.times { match_conf['type'] }
+    assert_equal before_size, match_conf.unused.size
+  end
+
+  def write_config(path, data, encoding: 'utf-8')
+    FileUtils.mkdir_p(File.dirname(path))
+    File.open(path, "w:#{encoding}:utf-8") {|f| f.write data }
+  end
+
+  def test_inline
+    prepare_config
+    opts = {
+      :config_path => "#{TMP_DIR}/config_test_1.conf",
+      :inline_config => "<source>\n  type http\n  port 2222\n </source>",
+      :use_v1_config => false
+    }
+    assert_nothing_raised do
+      Fluent::Supervisor.new(opts)
+    end
+    create_warn_dummy_logger
+  end
+
+  def create_warn_dummy_logger
+    dl_opts = {}
+    dl_opts[:log_level] = ServerEngine::DaemonLogger::WARN
+    logdev = Fluent::Test::DummyLogDevice.new
+    logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)
+    $log = Fluent::Log.new(logger)
+  end
+
+  sub_test_case '.build' do
+    test 'read config' do
+      write_config("#{TMP_DIR}/build/config_build.conf", 'key value')
+      c = Fluent::Config.build(config_path: "#{TMP_DIR}/build/config_build.conf")
+      assert_equal('value', c['key'])
+    end
+
+    test 'read config with encoding' do
+      write_config("#{TMP_DIR}/build/config_build2.conf", "#てすと\nkey value", encoding: 'shift_jis')
+
+      c = Fluent::Config.build(config_path: "#{TMP_DIR}/build/config_build2.conf", encoding: 'shift_jis')
+      assert_equal('value', c['key'])
+    end
+
+    test 'read config with additional_config' do
+      write_config("#{TMP_DIR}/build/config_build2.conf", "key value")
+
+      c = Fluent::Config.build(config_path: "#{TMP_DIR}/build/config_build2.conf", additional_config: 'key2 value2')
+      assert_equal('value', c['key'])
+      assert_equal('value2', c['key2'])
+    end
+  end
+end

data/test/test_configdsl.rb

@@ -0,0 +1,148 @@
+require_relative 'helper'
+require 'fluent/config/dsl'
+require 'fluent/test'
+
+class ConfigDSLTest < Test::Unit::TestCase
+  # TEST_CONFIG1 = %[
+  #   <source>
+  #     type forward
+  #     port 24224
+  #   </source>
+  #   <match test.**>
+  #     type forward
+  #     flush_interval 1s
+  #     <server>
+  #       host host0.example.com
+  #       port 24224
+  #     </server>
+  #     <server>
+  #       host host1.example.com
+  #       port 24224
+  #     </server>
+  #   </match>
+  # ]
+  TEST_DSL_CONFIG1 = %q[
+source {
+  type "forward"
+  port 24224
+}
+match('test.**') {
+  type "forward"
+  flush_interval "1s"
+  (0..1).each do |i|
+    server {
+      host "host#{i}.example.com"
+      port 24224
+    }
+  end
+}
+]
+
+  TEST_DSL_CONFIG2 = %q[
+v = [0, 1, 2]
+]
+
+  TEST_DSL_CONFIG3 = %q[
+match
+]
+
+  TEST_DSL_CONFIG4 = %q[
+match('aa', 'bb'){
+  type :null
+}
+]
+
+  TEST_DSL_CONFIG5 = %q[
+match('aa')
+]
+
+  def test_parse
+    root = Fluent::Config::DSL::Parser.parse(TEST_DSL_CONFIG1)
+
+    assert_equal 0, root.keys.size
+    assert_equal 2, root.elements.size
+
+    e0 = root.elements[0]
+    assert_equal 'source', e0.name
+    assert_equal '', e0.arg
+    assert_equal 'forward', e0['@type']
+    assert_equal '24224', e0['port']
+
+    e1 = root.elements[1]
+    assert_equal 'match', e1.name
+    assert_equal 'test.**', e1.arg
+    assert_equal 'forward', e1['@type']
+    assert_equal '1s', e1['flush_interval']
+    assert_equal 2, e1.elements.size
+    e1s0 = e1.elements[0]
+    assert_equal 'server', e1s0.name
+    assert_equal 'host0.example.com', e1s0['host']
+    assert_equal '24224', e1s0['port']
+    e1s1 = e1.elements[1]
+    assert_equal 'server', e1s1.name
+    assert_equal 'host1.example.com', e1s1['host']
+    assert_equal '24224', e1s1['port']
+  end
+
+  def test_parse2
+    root = Fluent::Config::DSL::Parser.parse(TEST_DSL_CONFIG2)
+
+    assert_equal 0, root.keys.size
+    assert_equal 0, root.elements.size
+  end
+
+  def test_config_error
+    assert_raise(ArgumentError) {
+      Fluent::Config::DSL::Parser.parse(TEST_DSL_CONFIG3)
+    }
+
+    assert_raise(ArgumentError) {
+      Fluent::Config::DSL::Parser.parse(TEST_DSL_CONFIG4)
+    }
+
+    assert_raise(ArgumentError) {
+      Fluent::Config::DSL::Parser.parse(TEST_DSL_CONFIG5)
+    }
+  end
+
+  def test_with_ruby_keyword
+    uname_string = `uname -a`
+    root1 = Fluent::Config::DSL::Parser.parse(<<DSL)
+uname_str = ruby.open('|uname -a'){|out| out.read}
+source {
+  uname uname_str
+}
+DSL
+    source1 = root1.elements.first
+    assert_equal 'source', source1.name
+    assert_equal 1, source1.keys.size
+    assert_equal uname_string, source1['uname']
+
+    root2 = Fluent::Config::DSL::Parser.parse(<<DSL)
+ruby_version = ruby {
+  require 'erb'
+  ERB.new('<%= RUBY_VERSION %> from erb').result
+}
+source {
+  version ruby_version
+}
+DSL
+    source2 = root2.elements.first
+    assert_equal 'source', source2.name
+    assert_equal 1, source2.keys.size
+    assert_equal "#{RUBY_VERSION} from erb", source2['version']
+
+    # Parser#parse raises NoMethodError when configuration dsl elements are written in ruby block
+    conf3 = <<DSL
+ruby {
+  source {
+    type "tail"
+  }
+}
+source {
+  uname uname_str
+}
+DSL
+    assert_raise (NoMethodError) { Fluent::Config::DSL::Parser.parse(conf3) }
+  end
+end

data/test/test_daemonizer.rb

@@ -0,0 +1,91 @@
+require_relative 'helper'
+require 'fluent/daemonizer'
+
+class DaemonizerTest < ::Test::Unit::TestCase
+  TMP_DIR = File.join(File.dirname(__FILE__), 'tmp', 'daemonizer')
+
+  setup do
+    FileUtils.mkdir_p(TMP_DIR)
+  end
+
+  teardown do
+    FileUtils.rm_rf(TMP_DIR) rescue nil
+  end
+
+  test 'makes pid file' do
+    pid_path = File.join(TMP_DIR, 'file.pid')
+
+    mock(Process).daemon(anything, anything).once
+    r = Fluent::Daemonizer.daemonize(pid_path) { 'ret' }
+    assert_equal 'ret', r
+    assert File.exist?(pid_path)
+    assert Process.pid.to_s, File.read(pid_path).to_s
+  end
+
+  test 'in platforms which do not support fork' do
+    pid_path = File.join(TMP_DIR, 'file.pid')
+
+    mock(Process).daemon(anything, anything) { raise NotImplementedError }
+    args = ['-c', 'test.conf']
+    mock(Process).spawn(anything, *args) { Process.pid }
+
+    Fluent::Daemonizer.daemonize(pid_path, args) { 'ret' }
+    assert File.exist?(pid_path)
+    assert Process.pid.to_s, File.read(pid_path).to_s
+  end
+
+  sub_test_case 'when pid file already exists' do
+    test 'raise an error when process is running' do
+      omit 'chmod of file does not affetct root user' if Process.uid.zero?
+      pid_path = File.join(TMP_DIR, 'file.pid')
+      File.write(pid_path, '1')
+
+      mock(Process).daemon(anything, anything).never
+      mock(Process).kill(0, 1).once
+
+      assert_raise(Fluent::ConfigError.new('pid(1) is running')) do
+        Fluent::Daemonizer.daemonize(pid_path) { 'ret' }
+      end
+    end
+
+    test 'raise an error when file is not redable' do
+      omit 'chmod of file does not affetct root user' if Process.uid.zero?
+      not_readable_path = File.join(TMP_DIR, 'not_readable.pid')
+
+      File.write(not_readable_path, '1')
+      FileUtils.chmod(0333, not_readable_path)
+
+      mock(Process).daemon(anything, anything).never
+      assert_raise(Fluent::ConfigError.new("Cannot access pid file: #{File.absolute_path(not_readable_path)}")) do
+        Fluent::Daemonizer.daemonize(not_readable_path) { 'ret' }
+      end
+    end
+
+    test 'raise an error when file is not writable' do
+      omit 'chmod of file does not affetct root user' if Process.uid.zero?
+      not_writable_path = File.join(TMP_DIR, 'not_writable.pid')
+
+      File.write(not_writable_path, '1')
+      FileUtils.chmod(0555, not_writable_path)
+
+      mock(Process).daemon(anything, anything).never
+      assert_raise(Fluent::ConfigError.new("Cannot access pid file: #{File.absolute_path(not_writable_path)}")) do
+        Fluent::Daemonizer.daemonize(not_writable_path) { 'ret' }
+      end
+    end
+
+    test 'raise an error when directory is not writable' do
+      omit 'chmod of file does not affetct root user' if Process.uid.zero?
+      not_writable_dir = File.join(TMP_DIR, 'not_writable')
+      pid_path = File.join(not_writable_dir, 'file.pid')
+
+      FileUtils.mkdir_p(not_writable_dir)
+      FileUtils.chmod(0555, not_writable_dir)
+
+      mock(Process).daemon(anything, anything).never
+      assert_raise(Fluent::ConfigError.new("Cannot access directory for pid file: #{File.absolute_path(not_writable_dir)}")) do
+        Fluent::Daemonizer.daemonize(pid_path) { 'ret' }
+      end
+    end
+  end
+end

data/test/test_engine.rb

@@ -0,0 +1,203 @@
+require_relative 'helper'
+require 'fluent/engine'
+require 'fluent/config'
+require 'fluent/input'
+require 'fluent/system_config'
+
+class EngineTest < ::Test::Unit::TestCase
+  class DummyEngineTestOutput < Fluent::Plugin::Output
+    Fluent::Plugin.register_output('dummy_engine_test', self)
+    def write(chunk); end
+  end
+
+  class DummyEngineTest2Output < Fluent::Plugin::Output
+    Fluent::Plugin.register_output('dummy_engine_test2', self)
+    def write(chunk); end
+  end
+
+  class DummyEngineTestInput < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('dummy_engine_test', self)
+    def multi_workers_ready?; true; end
+  end
+
+  class DummyEngineTest2Input < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('dummy_engine_test2', self)
+    def multi_workers_ready?; true; end
+  end
+
+  class DummyEngineClassVarTestInput < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('dummy_engine_class_var_test', self)
+    @@test = nil
+    def multi_workers_ready?; true; end
+  end
+
+  sub_test_case '#reload_config' do
+    test 'reload new configuration' do
+      conf_data = <<-CONF
+        <source>
+          @type dummy_engine_test
+        </source>
+        <match>
+          @type dummy_engine_test
+        </match>
+      CONF
+
+      conf = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
+      system_config = Fluent::SystemConfig.create(conf)
+
+      engine = Fluent::EngineClass.new
+      engine.init(system_config)
+      engine.configure(conf)
+
+      assert_kind_of DummyEngineTestInput, engine.root_agent.inputs[0]
+      assert_kind_of DummyEngineTestOutput, engine.root_agent.outputs[0]
+
+      new_conf_data = <<-CONF
+        <source>
+          @type dummy_engine_test2
+        </source>
+        <match>
+          @type dummy_engine_test2
+        </match>
+      CONF
+
+      new_conf = Fluent::Config.parse(new_conf_data, '(test)', '(test_dir)', true)
+
+      agent = Fluent::RootAgent.new(log: $log, system_config: system_config)
+      stub(Fluent::RootAgent).new do
+        stub(agent).start.once
+        agent
+      end
+
+      engine.reload_config(new_conf)
+
+      assert_kind_of DummyEngineTest2Input, engine.root_agent.inputs[0]
+      assert_kind_of DummyEngineTest2Output, engine.root_agent.outputs[0]
+    end
+
+    test "doesn't start RootAgent when supervisor is true" do
+      conf_data = <<-CONF
+        <source>
+          @type dummy_engine_test
+        </source>
+        <match>
+          @type dummy_engine_test
+        </match>
+      CONF
+
+      conf = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
+      system_config = Fluent::SystemConfig.create(conf)
+
+      engine = Fluent::EngineClass.new
+      engine.init(system_config)
+      engine.configure(conf)
+
+      assert_kind_of DummyEngineTestInput, engine.root_agent.inputs[0]
+      assert_kind_of DummyEngineTestOutput, engine.root_agent.outputs[0]
+
+      new_conf_data = <<-CONF
+        <source>
+          @type dummy_engine_test2
+        </source>
+        <match>
+          @type dummy_engine_test2
+        </match>
+      CONF
+
+      new_conf = Fluent::Config.parse(new_conf_data, '(test)', '(test_dir)', true)
+
+      agent = Fluent::RootAgent.new(log: $log, system_config: system_config)
+      stub(Fluent::RootAgent).new do
+        stub(agent).start.never
+        agent
+      end
+
+      engine.reload_config(new_conf, supervisor: true)
+
+      assert_kind_of DummyEngineTest2Input, engine.root_agent.inputs[0]
+      assert_kind_of DummyEngineTest2Output, engine.root_agent.outputs[0]
+    end
+
+    test 'raise an error when conf is invalid' do
+      conf_data = <<-CONF
+        <source>
+          @type dummy_engine_test
+        </source>
+        <match>
+          @type dummy_engine_test
+        </match>
+      CONF
+
+      conf = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
+      system_config = Fluent::SystemConfig.create(conf)
+
+      engine = Fluent::EngineClass.new
+      engine.init(system_config)
+      engine.configure(conf)
+
+      assert_kind_of DummyEngineTestInput, engine.root_agent.inputs[0]
+      assert_kind_of DummyEngineTestOutput, engine.root_agent.outputs[0]
+
+      new_conf_data = <<-CONF
+        <source>
+          @type
+        </source>
+      CONF
+
+      new_conf = Fluent::Config.parse(new_conf_data, '(test)', '(test_dir)', true)
+
+      agent = Fluent::RootAgent.new(log: $log, system_config: system_config)
+      stub(Fluent::RootAgent).new do
+        stub(agent).start.never
+        agent
+      end
+
+      assert_raise(Fluent::ConfigError.new("Missing '@type' parameter on <source> directive")) do
+        engine.reload_config(new_conf)
+      end
+
+      assert_kind_of DummyEngineTestInput, engine.root_agent.inputs[0]
+      assert_kind_of DummyEngineTestOutput, engine.root_agent.outputs[0]
+    end
+
+    test 'raise an error when unreloadable exists' do
+      conf_data = <<-CONF
+        <source>
+          @type dummy_engine_test
+        </source>
+        <match>
+          @type dummy_engine_test
+        </match>
+      CONF
+
+      conf = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
+      system_config = Fluent::SystemConfig.create(conf)
+
+      engine = Fluent::EngineClass.new
+      engine.init(system_config)
+      engine.configure(conf)
+
+      assert_kind_of DummyEngineTestInput, engine.root_agent.inputs[0]
+      assert_kind_of DummyEngineTestOutput, engine.root_agent.outputs[0]
+
+      conf_data = <<-CONF
+        <source>
+          @type dummy_engine_class_var_test
+        </source>
+        <match>
+          @type dummy_engine_test
+        </match>
+      CONF
+
+      new_conf = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
+
+      e = assert_raise(Fluent::ConfigError) do
+        engine.reload_config(new_conf)
+      end
+      assert e.message.match?('Unreloadable plugin plugin: dummy_engine_class_var_test')
+
+      assert_kind_of DummyEngineTestInput, engine.root_agent.inputs[0]
+      assert_kind_of DummyEngineTestOutput, engine.root_agent.outputs[0]
+    end
+  end
+end