fluentd 1.7.4-x64-mingw32 → 1.8.0-x64-mingw32

data/lib/fluent/system_config.rb

@@ -27,11 +27,12 @@ module Fluent
       :log_event_verbose,
       :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
       :file_permission, :dir_permission, :counter_server, :counter_client,
+      :strict_config_value
     ]
 
     config_param :workers,   :integer, default: 1
     config_param :root_dir,  :string, default: nil
-    config_param :log_level, :enum, list: [:trace, :debug, :info, :warn, :error, :fatal], default: nil
+    config_param :log_level, :enum, list: [:trace, :debug, :info, :warn, :error, :fatal], default: 'info'
     config_param :suppress_repeated_stacktrace, :bool, default: nil
     config_param :emit_error_log_interval,      :time, default: nil
     config_param :suppress_config_dump, :bool, default: nil
@@ -40,6 +41,7 @@ module Fluent
     config_param :rpc_endpoint,    :string, default: nil
     config_param :enable_get_dump, :bool, default: nil
     config_param :process_name,    :string, default: nil
+    config_param :strict_config_value, :bool, default: nil
     config_param :file_permission, default: nil do |v|
       v.to_i(8)
     end
@@ -73,12 +75,12 @@ module Fluent
       config_param :timeout, :time, default: nil
     end
 
-    def self.create(conf)
+    def self.create(conf, strict_config_value=false)
       systems = conf.elements(name: 'system')
       return SystemConfig.new if systems.empty?
       raise Fluent::ConfigError, "<system> is duplicated. <system> should be only one" if systems.size > 1
 
-      SystemConfig.new(systems.first)
+      SystemConfig.new(systems.first, strict_config_value)
     end
 
     def self.blank_system_config
@@ -95,14 +97,26 @@ module Fluent
       end
     end
 
-    def initialize(conf=nil)
+    def initialize(conf=nil, strict_config_value=false)
       super()
       conf ||= SystemConfig.blank_system_config
-      configure(conf)
+      configure(conf, strict_config_value)
     end
 
-    def configure(conf)
-      super
+    def configure(conf, strict_config_value=false)
+      strict = strict_config_value
+      if !strict && conf && conf.has_key?("strict_config_value")
+        strict = Fluent::Config.bool_value(conf["strict_config_value"])
+      end
+
+      begin
+        super(conf, strict)
+      rescue ConfigError => e
+        $log.error "config error in:\n#{conf}"
+        $log.error 'config error', error: e
+        $log.debug_backtrace
+        exit!(1)
+      end
 
       @log_level = Log.str_to_level(@log_level.to_s) if @log_level
     end
@@ -115,52 +129,12 @@ module Fluent
       s
     end
 
-    def attach(supervisor)
-      system = self
-      supervisor.instance_eval {
-        SYSTEM_CONFIG_PARAMETERS.each do |param|
-          case param
-          when :rpc_endpoint, :enable_get_dump, :process_name, :file_permission, :dir_permission, :counter_server, :counter_client
-            next # doesn't exist in command line options
-          when :emit_error_log_interval
-            system.emit_error_log_interval = @suppress_interval if @suppress_interval
-          when :log_level
-            ll_value = instance_variable_get("@log_level")
-            # info level can't be specified via command line option.
-            # log_level is info here, it is default value and <system>'s log_level should be applied if exists.
-            if ll_value != Fluent::Log::LEVEL_INFO
-              system.log_level = ll_value
-            end
-          else
-            next unless instance_variable_defined?("@#{param}")
-            supervisor_value = instance_variable_get("@#{param}")
-            next if supervisor_value.nil? # it's not configured by command line options
-
-            system.__send__("#{param}=", supervisor_value)
-          end
-        end
-      }
-    end
-
-    def apply(supervisor)
-      system = self
-      supervisor.instance_eval {
-        SYSTEM_CONFIG_PARAMETERS.each do |param|
-          param_value = system.__send__(param)
-          next if param_value.nil?
-
-          case param
-          when :log_level
-            @log.level = @log_level = param_value
-          when :emit_error_log_interval
-            @suppress_interval = param_value
-          else
-            instance_variable_set("@#{param}", param_value)
-          end
+    def overwrite_variables(**opt)
+      SYSTEM_CONFIG_PARAMETERS.each do |param|
+        if opt.key?(param) && !opt[param].nil? && instance_variable_defined?("@#{param}")
+          instance_variable_set("@#{param}", opt[param])
         end
-        #@counter_server = system.counter_server unless system.counter_server.nil?
-        #@counter_client = system.counter_client unless system.counter_client.nil?
-      }
+      end
     end
 
     module Mixin

data/lib/fluent/test/driver/base_owned.rb

@@ -53,13 +53,26 @@ module Fluent
           @section_name = ''
         end
 
-        def configure(conf, syntax: :v1)
+        def configure(conf)
           if conf.is_a?(Fluent::Config::Element)
             @config = conf
           elsif conf.is_a?(Hash)
             @config = Fluent::Config::Element.new(@section_name, "", Hash[conf.map{|k,v| [k.to_s, v]}], [])
           else
-            @config = Fluent::Config.parse(conf, @section_name, "", syntax: syntax)
+            @config = Fluent::Config.parse(conf, @section_name, "", syntax: :v1)
+          end
+          @instance.configure(@config)
+          self
+        end
+
+        # this is special method for v0 and should be deleted
+        def configure_v0(conf)
+          if conf.is_a?(Fluent::Config::Element)
+            @config = conf
+          elsif conf.is_a?(Hash)
+            @config = Fluent::Config::Element.new(@section_name, "", Hash[conf.map{|k,v| [k.to_s, v]}], [])
+          else
+            @config = Fluent::Config.parse(conf, @section_name, "", syntax: :v0)
           end
           @instance.configure(@config)
           self

data/lib/fluent/time.rb

@@ -147,14 +147,12 @@ module Fluent
     module TimeParameters
       include Fluent::Configurable
       TIME_FULL_PARAMETERS.each do |name, type, opts|
-        config_param name, type, opts
+        config_param(name, type, **opts)
       end
 
       def configure(conf)
         if conf.has_key?('localtime') || conf.has_key?('utc')
-          if conf.has_key?('localtime') && conf.has_key?('utc')
-            raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
-          elsif conf.has_key?('localtime')
+          if conf.has_key?('localtime')
             conf['localtime'] = Fluent::Config.bool_value(conf['localtime'])
           elsif conf.has_key?('utc')
             conf['localtime'] = !(Fluent::Config.bool_value(conf['utc']))
@@ -167,6 +165,10 @@ module Fluent
 
         super
 
+        if conf.has_key?('localtime') && conf.has_key?('utc') && !(@localtime ^ @utc)
+          raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
+        end
+
         Fluent::Timezone.validate!(@timezone) if @timezone
       end
     end
@@ -292,7 +294,7 @@ module Fluent
 
     def parse_unixtime(value)
       unless value.is_a?(String) || value.is_a?(Numeric)
-        raise TimeParseError, "value must be a string or a number: #{value}(value.class)"
+        raise TimeParseError, "value must be a string or a number: #{value}(#{value.class})"
       end
 
       if @cache1_key == value
@@ -323,7 +325,7 @@ module Fluent
     ## parse_by_to_r  (msec): 28.232856 sec
     def parse_float(value)
       unless value.is_a?(String) || value.is_a?(Numeric)
-        raise TimeParseError, "value must be a string or a number: #{value}(value.class)"
+        raise TimeParseError, "value must be a string or a number: #{value}(#{value.class})"
       end
 
       if @cache1_key == value

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.7.4'
+  VERSION = '1.8.0'
 
 end

data/test/command/test_fluentd.rb

@@ -317,7 +317,8 @@ CONF
         create_cmdline(conf_path),
         "fluentd worker is now running",
         'fluent.info: {"worker":0,"message":"fluentd worker is now running worker=0"}',
-        patterns_not_match: ['[warn]: some tags for log events are not defined (to be ignored) tags=["fluent.trace", "fluent.debug"]'],
+        "define <match fluent.**> to capture fluentd logs in top level is deprecated. Use <label @FLUENT_LOG> instead",
+        patterns_not_match: ['[warn]: some tags for log events are not defined in top level (to be ignored) tags=["fluent.trace", "fluent.debug"]'],
       )
     end
 
@@ -331,7 +332,8 @@ CONF
       assert_log_matches(
         create_cmdline(conf_path),
         "fluentd worker is now running",
-        '[warn]: #0 match for some tags of log events are not defined (to be ignored) tags=["fluent.trace", "fluent.debug", "fluent.info"]',
+        '[warn]: #0 match for some tags of log events are not defined in top level (to be ignored) tags=["fluent.trace", "fluent.debug", "fluent.info"]',
+        "define <match fluent.warn>, <match fluent.error>, <match fluent.fatal> to capture fluentd logs in top level is deprecated. Use <label @FLUENT_LOG> instead",
         '[warn]: #0 no patterns matched tag="fluent.info"',
       )
     end
@@ -349,7 +351,7 @@ CONF
         create_cmdline(conf_path),
         "fluentd worker is now running",
         'fluent.info: {"worker":0,"message":"fluentd worker is now running worker=0"}',
-        patterns_not_match: ['[warn]: some tags for log events are not defined (to be ignored)'],
+        patterns_not_match: ['[warn]: some tags for log events are not defined in @FLUENT_LOG label (to be ignored)'],
       )
     end
 
@@ -368,7 +370,7 @@ CONF
       assert_log_matches(
         create_cmdline(conf_path),
         "fluentd worker is now running",
-        '[warn]: #0 match for some tags of log events are not defined (to be ignored) tags=["fluent.info", "fluent.fatal"]',
+        '[warn]: #0 match for some tags of log events are not defined in @FLUENT_LOG label (to be ignored) tags=["fluent.info", "fluent.fatal"]',
         patterns_not_match: ['[warn]: no patterns matched tag="fluent.info"'],
       )
     end
@@ -625,7 +627,8 @@ CONF
   workers 2
 </system>
 <source>
-  @type single
+  @type dummy
+  tag dummy
   @id single
   @label @dummydata
 </source>
@@ -669,7 +672,8 @@ EOC
   workers 2
 </system>
 <source>
-  @type single
+  @type dummy
+  tag dummy
   @id single
   @label @dummydata
 </source>
@@ -793,7 +797,7 @@ CONF
     @id   blackhole
     <buffer>
       @type file
-      path #{File.join(@root_path, "buf", "file.*.log")}
+      path #{File.join(@root_path, "buf")}
     </buffer>
   </match>
 </worker>
@@ -863,6 +867,7 @@ module Fluent::Plugin
   class FakeInput < Input
     Fluent::Plugin.register_input('fake', self)
     config_param :secret, :string, secret: true
+    def multi_workers_ready?; true; end
   end
 end
 EOC

data/test/config/test_configurable.rb

@@ -30,6 +30,13 @@ module ConfigurableSpec
     config_set_default :opt1, :baz
   end
 
+  class Base1Nil < Base1
+    config_set_default :name1, nil
+    config_set_default :name2, nil
+    config_set_default :opt1, nil
+    config_param :name5, :string, default: nil
+  end
+
   class Base2 < Base1
     config_set_default :name2, "base2"
     config_set_default :name4, "base2"
@@ -214,6 +221,12 @@ module ConfigurableSpec
     end
   end
 
+  class ExampleWithIntFloat
+    include Fluent::Configurable
+    config_param :int1, :integer
+    config_param :float1, :float
+  end
+
   module Overwrite
     class Base
       include Fluent::Configurable
@@ -602,7 +615,69 @@ module Fluent::Config
             assert_equal([], x6a.obj2)
           end
         end
+
+        test 'strict value type' do
+          default = config_element("", "", {"int1" => "1", "float1" => ""})
+
+          c = ConfigurableSpec::ExampleWithIntFloat.new
+          assert_nothing_raised { c.configure(default) }
+          assert_raise(Fluent::ConfigError) { c.configure(default, true) }
+        end
       end
+
+        test 'set nil for a parameter which has no default value' do
+          obj = ConfigurableSpec::Base2.new
+          conf = config_element("", "", {"name1" => nil, "name5" => "t5", "opt3" => "a"})
+          assert_raise(Fluent::ConfigError.new("'name1' parameter is required but nil is specified")) do
+            obj.configure(conf)
+          end
+        end
+
+        test 'set nil for a parameter which has non-nil default value' do
+          obj = ConfigurableSpec::Base2.new
+          conf = config_element("", "", {"name1" => "t1", "name3" => nil, "name5" => "t5", "opt3" => "a"})
+          assert_raise(Fluent::ConfigError.new("'name3' parameter is required but nil is specified")) do
+            obj.configure(conf)
+          end
+        end
+
+        test 'set nil for a parameter whose default value is nil' do
+          obj = ConfigurableSpec::Base1Nil.new
+          conf = config_element("", "", {"name5" => nil})
+          obj.configure(conf)
+          assert_nil obj.name5
+        end
+
+        test 'set nil for parameters whose default values are overwritten by nil' do
+          obj = ConfigurableSpec::Base1Nil.new
+          conf = config_element("", "", {"name1" => nil, "name2" => nil, "opt1" => nil})
+          obj.configure(conf)
+          assert_nil obj.name1
+          assert_nil obj.name2
+          assert_nil obj.opt1
+        end
+
+        test 'set :default' do
+          obj = ConfigurableSpec::Base2.new
+          conf = config_element("", "", {"name1" => "t1", "name3" => :default, "name5" => "t5", "opt3" => "a"})
+          obj.configure(conf)
+          assert_equal "base1", obj.name3
+        end
+
+        test 'set :default for a parameter which has no default value' do
+          obj = ConfigurableSpec::Base2.new
+          conf = config_element("", "", {"name1" => :default, "name5" => "t5", "opt3" => "a"})
+          assert_raise(Fluent::ConfigError.new("'name1' doesn't have default value")) do
+            obj.configure(conf)
+          end
+        end
+
+        test 'set :default for a parameter which has an overwritten default value' do
+          obj = ConfigurableSpec::Base2.new
+          conf = config_element("", "", {"name1" => "t1", "name3" => "t3", "name4" => :default, "name5" => "t5", "opt3" => "a"})
+          obj.configure(conf)
+          assert_equal "base2", obj.name4
+        end
     end
 
     sub_test_case 'class defined with config_section' do
@@ -1626,5 +1701,84 @@ module Fluent::Config
         assert_equal 'foo', c.subsection.first.param0
       end
     end
+
+    sub_test_case '#config_argument' do
+      test 'with strict_config_value' do
+        class TestClass01
+          include Fluent::Configurable
+          config_section :subsection do
+            config_argument :param1, :integer
+          end
+        end
+
+        c = TestClass01.new
+        subsection = config_element('subsection', "hoge", { })
+        assert_raise(Fluent::ConfigError.new('param1: invalid value for Integer(): "hoge"')) do
+          c.configure(config_element('root', '', {}, [subsection]), true)
+        end
+      end
+
+      test 'with nil' do
+        class TestClass02
+          include Fluent::Configurable
+          config_section :subsection do
+            config_argument :param1, :integer
+          end
+        end
+
+        c = TestClass02.new
+        subsection = config_element('subsection', nil, { })
+        assert_raise(Fluent::ConfigError.new("'<subsection ARG>' section requires argument, in section subsection")) do
+          c.configure(config_element('root', '', {}, [subsection]))
+        end
+      end
+
+      test 'with nil for an argument whose default value is nil' do
+        class TestClass03
+          include Fluent::Configurable
+          config_section :subsection do
+            config_argument :param1, :integer, default: nil
+          end
+        end
+
+        c = TestClass03.new
+        subsection = config_element('subsection', nil, { })
+        c.configure(config_element('root', '', {}, [subsection]))
+
+        assert_equal 1, c.subsection.size
+        assert_equal nil, c.subsection.first.param1
+      end
+
+      test 'with :default' do
+        class TestClass04
+          include Fluent::Configurable
+          config_section :subsection do
+            config_argument :param1, :integer, default: 3
+          end
+        end
+
+        c = TestClass04.new
+        subsection = config_element('subsection', :default, { })
+        c.configure(config_element('root', '', {}, [subsection]))
+
+        assert_equal 1, c.subsection.size
+        assert_equal 3, c.subsection.first.param1
+      end
+
+      test 'with :default for an argument which does not have default value' do
+        class TestClass05
+          include Fluent::Configurable
+          config_section :subsection do
+            config_argument :param1, :integer
+          end
+        end
+
+        c = TestClass05.new
+        subsection = config_element('subsection', :default, { })
+        assert_raise(Fluent::ConfigError.new("'param1' doesn\'t have default value")) do
+          c.configure(config_element('root', '', {}, [subsection]))
+        end
+      end
+    end
   end
 end