fluentd222 1.16.2-x86_64-linux

data/test/config/test_system_config.rb

@@ -0,0 +1,195 @@
+require_relative '../helper'
+require 'fluent/configurable'
+require 'fluent/config/element'
+require 'fluent/config/section'
+require 'fluent/system_config'
+
+module Fluent::Config
+  class FakeSupervisor
+    attr_writer :log_level
+
+    def initialize(**opt)
+      @system_config = nil
+      @cl_opt = {
+        wokers: nil,
+        restart_worker_interval: nil,
+        root_dir: nil,
+        log_level: Fluent::Log::LEVEL_INFO,
+        suppress_interval: nil,
+        suppress_config_dump: nil,
+        suppress_repeated_stacktrace: nil,
+        log_event_label: nil,
+        log_event_verbose: nil,
+        without_source: nil,
+        enable_input_metrics: nil,
+        enable_size_metrics: nil,
+        emit_error_log_interval: nil,
+        file_permission: nil,
+        dir_permission: nil,
+      }.merge(opt)
+    end
+
+    def for_system_config
+      opt = {}
+      # this is copy from Supervisor#build_system_config
+      Fluent::SystemConfig::SYSTEM_CONFIG_PARAMETERS.each do |param|
+        if @cl_opt.key?(param) && !@cl_opt[param].nil?
+          if param == :log_level && @cl_opt[:log_level] == Fluent::Log::LEVEL_INFO
+            # info level can't be specified via command line option.
+            # log_level is info here, it is default value and <system>'s log_level should be applied if exists.
+            next
+          end
+
+          opt[param] = @cl_opt[param]
+        end
+      end
+
+      opt
+    end
+  end
+
+  class TestSystemConfig < ::Test::Unit::TestCase
+    TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/tmp/system_config/#{ENV['TEST_ENV_NUMBER']}")
+
+    def parse_text(text)
+      basepath = File.expand_path(File.dirname(__FILE__) + '/../../')
+      Fluent::Config.parse(text, '(test)', basepath, true).elements.find { |e| e.name == 'system' }
+    end
+
+    test 'should not override default configurations when no parameters' do
+      conf = parse_text(<<-EOS)
+        <system>
+        </system>
+      EOS
+      s = FakeSupervisor.new
+      sc = Fluent::SystemConfig.new(conf)
+      sc.overwrite_variables(**s.for_system_config)
+      assert_equal(1, sc.workers)
+      assert_equal(0, sc.restart_worker_interval)
+      assert_nil(sc.root_dir)
+      assert_equal(Fluent::Log::LEVEL_INFO, sc.log_level)
+      assert_nil(sc.suppress_repeated_stacktrace)
+      assert_nil(sc.ignore_repeated_log_interval)
+      assert_nil(sc.emit_error_log_interval)
+      assert_nil(sc.suppress_config_dump)
+      assert_nil(sc.without_source)
+      assert_nil(sc.enable_input_metrics)
+      assert_nil(sc.enable_size_metrics)
+      assert_nil(sc.enable_msgpack_time_support)
+      assert(!sc.enable_jit)
+      assert_equal(:text, sc.log.format)
+      assert_equal('%Y-%m-%d %H:%M:%S %z', sc.log.time_format)
+    end
+
+    data(
+      'workers' => ['workers', 3],
+      'restart_worker_interval' => ['restart_worker_interval', 60],
+      'root_dir' => ['root_dir', File.join(TMP_DIR, 'root')],
+      'log_level' => ['log_level', 'error'],
+      'suppress_repeated_stacktrace' => ['suppress_repeated_stacktrace', true],
+      'ignore_repeated_log_interval' => ['ignore_repeated_log_interval', 10],
+      'log_event_verbose' => ['log_event_verbose', true],
+      'suppress_config_dump' => ['suppress_config_dump', true],
+      'without_source' => ['without_source', true],
+      'strict_config_value' => ['strict_config_value', true],
+      'enable_msgpack_time_support' => ['enable_msgpack_time_support', true],
+      'enable_input_metrics' => ['enable_input_metrics', true],
+      'enable_size_metrics' => ['enable_size_metrics', true],
+      'enable_jit' => ['enable_jit', true],
+    )
+    test "accepts parameters" do |(k, v)|
+      conf = parse_text(<<-EOS)
+          <system>
+            #{k} #{v}
+          </system>
+      EOS
+      s = FakeSupervisor.new
+      sc = Fluent::SystemConfig.new(conf)
+      sc.overwrite_variables(**s.for_system_config)
+      if k == 'log_level'
+        assert_equal(Fluent::Log::LEVEL_ERROR, sc.__send__(k))
+      else
+        assert_equal(v, sc.__send__(k))
+      end
+    end
+
+    test "log parameters" do
+      conf = parse_text(<<-EOS)
+          <system>
+            <log>
+              format json
+              time_format %Y
+            </log>
+          </system>
+      EOS
+      s = FakeSupervisor.new
+      sc = Fluent::SystemConfig.new(conf)
+      sc.overwrite_variables(**s.for_system_config)
+      assert_equal(:json, sc.log.format)
+      assert_equal('%Y', sc.log.time_format)
+    end
+
+    # info is removed because info level can't be specified via command line
+    data('trace' => Fluent::Log::LEVEL_TRACE,
+         'debug' => Fluent::Log::LEVEL_DEBUG,
+         'warn' => Fluent::Log::LEVEL_WARN,
+         'error' => Fluent::Log::LEVEL_ERROR,
+         'fatal' => Fluent::Log::LEVEL_FATAL)
+    test 'log_level is ignored when log_level related command line option is passed' do |level|
+      conf = parse_text(<<-EOS)
+        <system>
+          log_level info
+        </system>
+      EOS
+      s = FakeSupervisor.new(log_level: level)
+      sc = Fluent::SystemConfig.new(conf)
+      sc.overwrite_variables(**s.for_system_config)
+      assert_equal(level, sc.log_level)
+    end
+
+    sub_test_case "log rotation" do
+      data('daily' => "daily",
+           'weekly' => 'weekly',
+           'monthly' => 'monthly')
+      test "strings for rotate_age" do |age|
+        conf = parse_text(<<-EOS)
+          <system>
+            <log>
+              rotate_age #{age}
+            </log>
+          </system>
+        EOS
+        sc = Fluent::SystemConfig.new(conf)
+        assert_equal(age, sc.log.rotate_age)
+      end
+
+      test "numeric number for rotate age" do
+        conf = parse_text(<<-EOS)
+          <system>
+            <log>
+              rotate_age 3
+            </log>
+          </system>
+        EOS
+        sc = Fluent::SystemConfig.new(conf)
+        assert_equal(3, sc.log.rotate_age)
+      end
+
+      data(h: ['100', 100],
+           k: ['1k', 1024],
+           m: ['1m', 1024 * 1024],
+           g: ['1g', 1024 * 1024 * 1024])
+      test "numeric and SI prefix for rotate_size" do |(label, size)|
+        conf = parse_text(<<-EOS)
+          <system>
+            <log>
+              rotate_size #{label}
+            </log>
+          </system>
+        EOS
+        sc = Fluent::SystemConfig.new(conf)
+        assert_equal(size, sc.log.rotate_size)
+      end
+    end
+  end
+end

data/test/config/test_types.rb

@@ -0,0 +1,408 @@
+require 'helper'
+require 'fluent/config/types'
+
+class TestConfigTypes < ::Test::Unit::TestCase
+  include Fluent
+
+  sub_test_case 'Config.size_value' do
+    data("2k" => [2048, "2k"],
+         "2K" => [2048, "2K"],
+         "3m" => [3145728, "3m"],
+         "3M" => [3145728, "3M"],
+         "4g" => [4294967296, "4g"],
+         "4G" => [4294967296, "4G"],
+         "5t" => [5497558138880, "5t"],
+         "5T" => [5497558138880, "5T"],
+         "6"  => [6, "6"])
+    test 'normal case' do |(expected, val)|
+      assert_equal(expected, Config.size_value(val))
+      assert_equal(expected, Config.size_value(val, { strict: true }))
+    end
+
+    data("integer" => [6, 6],
+         "hoge" => [0, "hoge"],
+         "empty" => [0, ""])
+    test 'not assumed case' do |(expected, val)|
+      assert_equal(expected, Config.size_value(val))
+    end
+
+    test 'nil' do
+      assert_equal(nil, Config.size_value(nil))
+    end
+
+    data("integer" => [6, 6],
+         "hoge" => [Fluent::ConfigError.new('name1: invalid value for Integer(): "hoge"'), "hoge"],
+         "empty" => [Fluent::ConfigError.new('name1: invalid value for Integer(): ""'), ""])
+    test 'not assumed case with strict' do |(expected, val)|
+      if expected.kind_of? Exception
+        assert_raise(expected) do
+          Config.size_value(val, { strict: true }, "name1")
+        end
+      else
+        assert_equal(expected, Config.size_value(val, { strict: true }, "name1"))
+      end
+    end
+
+    test 'nil with strict' do
+      assert_equal(nil, Config.size_value(nil, { strict: true }))
+    end
+  end
+
+  sub_test_case 'Config.time_value' do
+    data("10s" => [10, "10s"],
+         "10sec" => [10, "10sec"],
+         "2m" => [120, "2m"],
+         "3h" => [10800, "3h"],
+         "4d" => [345600, "4d"])
+    test 'normal case' do |(expected, val)|
+      assert_equal(expected, Config.time_value(val))
+      assert_equal(expected, Config.time_value(val, { strict: true }))
+    end
+
+    data("integer" => [4.0, 4],
+         "float" => [0.4, 0.4],
+         "hoge" => [0.0, "hoge"],
+         "empty" => [0.0, ""])
+    test 'not assumed case' do |(expected, val)|
+      assert_equal(expected, Config.time_value(val))
+    end
+
+    test 'nil' do
+      assert_equal(nil, Config.time_value(nil))
+    end
+
+    data("integer" => [6, 6],
+         "hoge" => [Fluent::ConfigError.new('name1: invalid value for Float(): "hoge"'), "hoge"],
+         "empty" => [Fluent::ConfigError.new('name1: invalid value for Float(): ""'), ""])
+    test 'not assumed case with strict' do |(expected, val)|
+      if expected.kind_of? Exception
+        assert_raise(expected) do
+          Config.time_value(val, { strict: true }, "name1")
+        end
+      else
+        assert_equal(expected, Config.time_value(val, { strict: true }, "name1"))
+      end
+    end
+
+    test 'nil with strict' do
+      assert_equal(nil, Config.time_value(nil, { strict: true }))
+    end
+  end
+
+  sub_test_case 'Config.bool_value' do
+    data("true" => [true, "true"],
+         "yes" => [true, "yes"],
+         "empty" => [true, ""],
+         "false" => [false, "false"],
+         "no" => [false, "no"])
+    test 'normal case' do |(expected, val)|
+      assert_equal(expected, Config.bool_value(val))
+    end
+
+    data("true" =>  [true,  true],
+         "false" => [false, false],
+         "hoge" => [nil, "hoge"],
+         "nil" => [nil, nil],
+         "integer" => [nil, 10])
+    test 'not assumed case' do |(expected, val)|
+      assert_equal(expected, Config.bool_value(val))
+    end
+
+    data("true" =>  [true,  true],
+         "false" => [false, false],
+         "hoge" => [Fluent::ConfigError.new("name1: invalid bool value: hoge"), "hoge"],
+         "nil" => [nil, nil],
+         "integer" => [Fluent::ConfigError.new("name1: invalid bool value: 10"), 10])
+    test 'not assumed case with strict' do |(expected, val)|
+      if expected.kind_of? Exception
+        assert_raise(expected) do
+          Config.bool_value(val, { strict: true }, "name1")
+        end
+      else
+        assert_equal(expected, Config.bool_value(val, { strict: true }, "name1"))
+      end
+    end
+  end
+
+  sub_test_case 'Config.regexp_value' do
+    data("empty" => [//, "//"],
+         "plain" => [/regexp/, "/regexp/"],
+         "zero width" => [/^$/, "/^$/"],
+         "character classes" => [/[a-z]/, "/[a-z]/"],
+         "meta charactersx" => [/.+.*?\d\w\s\S/, '/.+.*?\d\w\s\S/'])
+    test 'normal case' do |(expected, str)|
+      assert_equal(expected, Config.regexp_value(str))
+    end
+
+    data("empty" => [//, ""],
+         "plain" => [/regexp/, "regexp"],
+         "zero width" => [/^$/, "^$"],
+         "character classes" => [/[a-z]/, "[a-z]"],
+         "meta charactersx" => [/.+.*?\d\w\s\S/, '.+.*?\d\w\s\S'])
+    test 'w/o slashes' do |(expected, str)|
+      assert_equal(expected, Config.regexp_value(str))
+    end
+
+    data("missing right slash" => "/regexp",
+         "too many options" => "/regexp/imx",)
+    test 'invalid regexp' do |(str)|
+      assert_raise(Fluent::ConfigError.new("invalid regexp: missing right slash: #{str}")) do
+        Config.regexp_value(str)
+      end
+    end
+
+    test 'nil' do
+      assert_equal nil, Config.regexp_value(nil)
+    end
+  end
+
+  sub_test_case 'type converters for config_param definitions' do
+    data("test" => ['test', 'test'],
+         "1" =>  ['1',    '1'],
+         "spaces" =>  ['   ',  '   '])
+    test 'string' do |(expected, val)|
+      assert_equal expected, Config::STRING_TYPE.call(val, {})
+      assert_equal Encoding::UTF_8, Config::STRING_TYPE.call(val, {}).encoding
+    end
+
+    test 'string nil' do
+      assert_equal nil, Config::STRING_TYPE.call(nil, {})
+    end
+
+    data('latin' => 'Märch',
+         'ascii' => 'ascii',
+         'space' => '     ',
+         'number' => '1',
+         'Hiragana' => 'あいうえお')
+    test 'string w/ binary' do |str|
+      actual = Config::STRING_TYPE.call(str.b, {})
+      assert_equal str, actual
+      assert_equal Encoding::UTF_8, actual.encoding
+    end
+
+    data('starts_with_semicolon' => [:conor, ':conor'],
+         'simple_string' => [:conor, 'conor'],
+         'empty_string' => [nil, ''])
+    test 'symbol' do |(expected, val)|
+      assert_equal Config::SYMBOL_TYPE.call(val, {}), expected
+    end
+
+    data("val" => [:val, 'val'],
+         "v" => [:v, 'v'],
+         "value" => [:value, 'value'])
+    test 'enum' do |(expected, val)|
+      assert_equal expected, Config::ENUM_TYPE.call(val, {list: [:val, :value, :v]})
+    end
+
+    test 'enum: pick unknown choice' do
+      assert_raises(Fluent::ConfigError.new("valid options are val,value,v but got x")) do
+        Config::ENUM_TYPE.call('x', {list: [:val, :value, :v]})
+      end
+    end
+
+    data("empty list"  => {},
+         "string list" => {list: ["val", "value", "v"]})
+    test 'enum: invalid choices' do | list |
+      assert_raises(RuntimeError.new("Plugin BUG: config type 'enum' requires :list of symbols")) do
+        Config::ENUM_TYPE.call('val', list)
+      end
+    end
+
+    test 'enum: nil' do
+      assert_equal nil, Config::ENUM_TYPE.call(nil)
+    end
+
+    data("1" => [1, '1'],
+         "1.0" => [1, '1.0'],
+         "1_000" => [1000, '1_000'],
+         "1x" => [1, '1x'])
+    test 'integer' do |(expected, val)|
+      assert_equal expected, Config::INTEGER_TYPE.call(val, {})
+    end
+
+    data("integer" => [6, 6],
+         "hoge" => [0, "hoge"],
+         "empty" => [0, ""])
+    test 'integer: not assumed case' do |(expected, val)|
+      assert_equal expected, Config::INTEGER_TYPE.call(val, {})
+    end
+
+    test 'integer: nil' do
+      assert_equal nil, Config::INTEGER_TYPE.call(nil, {})
+    end
+
+    data("integer" => [6, 6],
+         "hoge" => [Fluent::ConfigError.new('name1: invalid value for Integer(): "hoge"'), "hoge"],
+         "empty" => [Fluent::ConfigError.new('name1: invalid value for Integer(): ""'), ""])
+    test 'integer: not assumed case with strict' do |(expected, val)|
+      if expected.kind_of? Exception
+        assert_raise(expected) do
+          Config::INTEGER_TYPE.call(val, { strict: true }, "name1")
+        end
+      else
+        assert_equal expected, Config::INTEGER_TYPE.call(val, { strict: true }, "name1")
+      end
+    end
+
+    test 'integer: nil with strict' do
+      assert_equal nil, Config::INTEGER_TYPE.call(nil, { strict: true })
+    end
+
+    data("1" => [1.0, '1'],
+         "1.0" => [1.0, '1.0'],
+         "1.00" => [1.0, '1.00'],
+         "1e0" => [1.0, '1e0'])
+    test 'float' do |(expected, val)|
+      assert_equal expected, Config::FLOAT_TYPE.call(val, {})
+    end
+
+    data("integer" => [6, 6],
+         "hoge" => [0, "hoge"],
+         "empty" => [0, ""])
+    test 'float: not assumed case' do |(expected, val)|
+      assert_equal expected, Config::FLOAT_TYPE.call(val, {})
+    end
+
+    test 'float: nil' do
+      assert_equal nil, Config::FLOAT_TYPE.call(nil, {})
+    end
+
+    data("integer" => [6, 6],
+         "hoge" => [Fluent::ConfigError.new('name1: invalid value for Float(): "hoge"'), "hoge"],
+         "empty" => [Fluent::ConfigError.new('name1: invalid value for Float(): ""'), ""])
+    test 'float: not assumed case with strict' do |(expected, val)|
+      if expected.kind_of? Exception
+        assert_raise(expected) do
+          Config::FLOAT_TYPE.call(val, { strict: true }, "name1")
+        end
+      else
+        assert_equal expected, Config::FLOAT_TYPE.call(val, { strict: true }, "name1")
+      end
+    end
+
+    test 'float: nil with strict' do
+      assert_equal nil, Config::FLOAT_TYPE.call(nil, { strict: true })
+    end
+
+    data("1000" => [1000, '1000'],
+         "1k" => [1024, '1k'],
+         "1m" => [1024*1024, '1m'])
+    test 'size' do |(expected, val)|
+      assert_equal expected, Config::SIZE_TYPE.call(val, {})
+    end
+
+    data("true" => [true, 'true'],
+         "yes" => [true, 'yes'],
+         "no" => [false, 'no'],
+         "false" => [false, 'false'],
+         "TRUE" => [nil, 'TRUE'],
+         "True" => [nil, 'True'],
+         "Yes" => [nil, 'Yes'],
+         "No" => [nil, 'No'],
+         "empty" => [true, ''],
+         "unexpected_string" => [nil, 'unexpected_string'])
+    test 'bool' do |(expected, val)|
+      assert_equal expected, Config::BOOL_TYPE.call(val, {})
+    end
+
+    data("0" => [0, '0'],
+         "1" => [1.0, '1'],
+         "1.01" => [1.01,  '1.01'],
+         "1s" => [1, '1s'],
+         "1," => [60, '1m'],
+         "1h" => [3600,  '1h'],
+         "1d" => [86400, '1d'])
+    test 'time' do |(expected, val)|
+      assert_equal expected, Config::TIME_TYPE.call(val, {})
+    end
+
+    data("empty" => [//, "//"],
+         "plain" => [/regexp/, "/regexp/"],
+         "zero width" => [/^$/, "/^$/"],
+         "character classes" => [/[a-z]/, "/[a-z]/"],
+         "meta charactersx" => [/.+.*?\d\w\s\S/, '/.+.*?\d\w\s\S/'])
+    test 'regexp' do |(expected, str)|
+      assert_equal(expected, Config::REGEXP_TYPE.call(str, {}))
+    end
+
+    data("string and integer" => [{"x"=>"v","k"=>1}, '{"x":"v","k":1}', {}],
+         "strings" => [{"x"=>"v","k"=>"1"}, 'x:v,k:1', {}],
+         "w/ space" => [{"x"=>"v","k"=>"1"}, 'x:v, k:1', {}],
+         "heading space" => [{"x"=>"v","k"=>"1"}, ' x:v, k:1 ', {}],
+         "trailing space" => [{"x"=>"v","k"=>"1"}, 'x:v , k:1 ', {}],
+         "multiple colons" => [{"x"=>"v:v","k"=>"1"}, 'x:v:v, k:1', {}],
+         "symbolize keys" => [{x: "v", k: 1}, '{"x":"v","k":1}', {symbolize_keys: true}],
+         "value_type: :string" => [{x: "v", k: "1"}, 'x:v,k:1', {symbolize_keys: true, value_type: :string}],
+         "value_type: :string 2" => [{x: "v", k: "1"}, '{"x":"v","k":1}', {symbolize_keys: true, value_type: :string}],
+         "value_type: :integer" => [{x: 0, k: 1}, 'x:0,k:1', {symbolize_keys: true, value_type: :integer}],
+         "time 1" => [{"x"=>1,"y"=>60,"z"=>3600}, '{"x":"1s","y":"1m","z":"1h"}', {value_type: :time}],
+         "time 2" => [{"x"=>1,"y"=>60,"z"=>3600}, 'x:1s,y:1m,z:1h', {value_type: :time}])
+    test 'hash' do |(expected, val, opts)|
+      assert_equal(expected, Config::HASH_TYPE.call(val, opts))
+    end
+
+    test 'hash w/ unknown type' do
+      assert_raise(RuntimeError.new("unknown type in REFORMAT: foo")) do
+        Config::HASH_TYPE.call("x:1,y:2", {value_type: :foo})
+      end
+    end
+
+    test 'hash w/ strict option' do
+      assert_raise(Fluent::ConfigError.new('y: invalid value for Integer(): "hoge"')) do
+        Config::HASH_TYPE.call("x:1,y:hoge", {value_type: :integer, strict: true})
+      end
+    end
+
+    data('latin' => ['3:Märch', {"3"=>"Märch"}],
+         'ascii' => ['ascii:ascii', {"ascii"=>"ascii"}],
+         'number' => ['number:1', {"number"=>"1"}],
+         'Hiragana' => ['hiragana:あいうえお', {"hiragana"=>"あいうえお"}])
+    test 'hash w/ binary' do |(target, expected)|
+      assert_equal(expected, Config::HASH_TYPE.call(target.b, { value_type: :string }))
+    end
+
+    test 'hash w/ nil' do
+      assert_equal(nil, Config::HASH_TYPE.call(nil))
+    end
+
+    data("strings and integer" => [["1","2",1],   '["1","2",1]', {}],
+         "number strings" => [["1","2","1"], '1,2,1', {}],
+         "alphabets" => [["a","b","c"], '["a","b","c"]', {}],
+         "alphabets w/o quote" => [["a","b","c"], 'a,b,c', {}],
+         "w/ spaces" => [["a","b","c"], 'a, b, c', {}],
+         "w/ space before comma" => [["a","b","c"], 'a , b , c', {}],
+         "comma or space w/ qupte" => [["a a","b,b"," c "], '["a a","b,b"," c "]', {}],
+         "space in a value w/o qupte" => [["a a","b","c"], 'a a,b,c', {}],
+         "integers" => [[1,2,1], '[1,2,1]', {}],
+         "value_type: :integer w/ quote" => [[1,2,1], '["1","2","1"]', {value_type: :integer}],
+         "value_type: :integer w/o quote" => [[1,2,1], '1,2,1', {value_type: :integer}])
+    test 'array' do |(expected, val, opts)|
+      assert_equal(expected, Config::ARRAY_TYPE.call(val, opts))
+    end
+
+    data('["1","2"]' => [["1","2"], '["1","2"]'],
+         '["3"]' => [["3"], '["3"]'])
+    test 'array w/ default values' do |(expected, val)|
+      array_options = {
+        default: [],
+      }
+      assert_equal(expected, Config::ARRAY_TYPE.call(val, array_options))
+    end
+
+    test 'array w/ unknown type' do
+      assert_raise(RuntimeError.new("unknown type in REFORMAT: foo")) do
+        Config::ARRAY_TYPE.call("1,2", {value_type: :foo})
+      end
+    end
+
+    test 'array w/ strict option' do
+      assert_raise(Fluent::ConfigError.new(': invalid value for Integer(): "hoge"')) do
+        Config::ARRAY_TYPE.call("1,hoge", {value_type: :integer, strict: true}, "name1")
+      end
+    end
+
+    test 'array w/ nil' do
+      assert_equal(nil, Config::ARRAY_TYPE.call(nil))
+    end
+  end
+end