fluentd 0.12.40 → 0.14.0

data/lib/fluent/config/dsl.rb

@@ -35,19 +35,14 @@ module Fluent
       end
 
       class Proxy
-        def initialize(name, arg, include_basepath = Dir.pwd)
+        def initialize(name, arg)
           @element = Element.new(name, arg, self)
-          @include_basepath = include_basepath
         end
 
         def element
           @element
         end
 
-        def include_basepath
-          @include_basepath
-        end
-
         def eval(source, source_path)
           @element.instance_eval(source, source_path)
           self
@@ -82,6 +77,10 @@ module Fluent
           @proxy    = proxy
         end
 
+        def to_int
+          __id__
+        end
+
         def method_missing(name, *args, &block)
           ::Kernel.raise ::ArgumentError, "Configuration DSL Syntax Error: only one argument allowed" if args.size > 1
           value = args.first
@@ -101,18 +100,6 @@ module Fluent
           self
         end
 
-        def include(*args)
-          ::Kernel.raise ::ArgumentError, "#{name} block requires arguments for include path" if args.nil? || args.size != 1
-          if args.first =~ /\.rb$/
-            path = File.expand_path(args.first)
-            data = File.read(path)
-            self.instance_eval(data, path)
-          else
-            ss = StringScanner.new('')
-            Config::V1Parser.new(ss, @proxy.include_basepath, '', nil).eval_include(@attrs, @elements, args.first)
-          end
-        end
-
         def source(&block)
           @proxy.add_element('source', nil, block)
         end

data/lib/fluent/config/element.rb

@@ -32,11 +32,36 @@ module Fluent
         @v1_config = false
         @corresponding_proxies = [] # some plugins use flat parameters, e.g. in_http doesn't provide <format> section for parser.
         @unused_in = false # if this element is not used in plugins, correspoing plugin name and parent element name is set, e.g. [source, plugin class].
+
+        # it's global logger, not plugin logger: deprecated message should be global warning, not plugin level.
+        @logger = defined?($log) ? $log : nil
       end
 
-      attr_accessor :name, :arg, :elements, :unused, :v1_config, :corresponding_proxies, :unused_in
+      attr_accessor :name, :arg, :unused, :v1_config, :corresponding_proxies, :unused_in
+      attr_writer :elements
+
+      RESERVED_PARAMETERS_COMPAT = {
+        '@type' => 'type',
+        '@id' => 'id',
+        '@log_level' => 'log_level',
+        '@label' => nil,
+      }
+      RESERVED_PARAMETERS = RESERVED_PARAMETERS_COMPAT.keys
+
+      def elements(*names, name: nil, arg: nil)
+        raise ArgumentError, "name and names are exclusive" if name && !names.empty?
+        raise ArgumentError, "arg is available only with name" if arg && !name
+
+        if name
+          @elements.select{|e| e.name == name && (!arg || e.arg == arg) }
+        elsif !names.empty?
+          @elements.select{|e| names.include?(e.name) }
+        else
+          @elements
+        end
+      end
 
-      def add_element(name, arg='')
+      def add_element(name, arg = '')
         e = Element.new(name, arg, {}, [])
         e.v1_config = @v1_config
         @elements << e
@@ -63,6 +88,7 @@ module Fluent
         e
       end
 
+      # no code in fluentd uses this method
       def each_element(*names, &block)
         if names.empty?
           @elements.each(&block)
@@ -84,6 +110,12 @@ module Fluent
       def [](key)
         @unused_in = false # ditto
         @unused.delete(key)
+
+        if RESERVED_PARAMETERS.include?(key) && !has_key?(key) && has_key?(RESERVED_PARAMETERS_COMPAT[key])
+          @logger.warn "'#{RESERVED_PARAMETERS_COMPAT[key]}' is deprecated parameter name. use '#{key}' instead." if @logger
+          return self[RESERVED_PARAMETERS_COMPAT[key]]
+        end
+
         super
       end
 
@@ -108,11 +140,7 @@ module Fluent
           out << "#{indent}<#{@name} #{@arg}>\n"
         end
         each_pair { |k, v|
-          if secret_param?(k)
-            out << "#{nindent}#{k} xxxxxx\n"
-          else
-            out << "#{nindent}#{k} #{v}\n"
-          end
+          out << dump_value(k, v, indent, nindent)
         }
         @elements.each { |e|
           out << e.to_s(nest + 1)
@@ -124,6 +152,8 @@ module Fluent
       def to_masked_element
         new_elems = @elements.map { |e| e.to_masked_element }
         new_elem = Element.new(@name, @arg, {}, new_elems, @unused)
+        new_elem.v1_config = @v1_config
+        new_elem.corresponding_proxies = @corresponding_proxies
         each_pair { |k, v|
           new_elem[k] = secret_param?(k) ? 'xxxxxx' : v
         }
@@ -135,7 +165,7 @@ module Fluent
 
         param_key = key.to_sym
         @corresponding_proxies.each { |proxy|
-          block, opts = proxy.params[param_key]
+          _block, opts = proxy.params[param_key]
           if opts && opts.has_key?(:secret)
             return opts[:secret]
           end
@@ -144,6 +174,40 @@ module Fluent
         false
       end
 
+      def param_type(key)
+        return nil if @corresponding_proxies.empty?
+
+        param_key = key.to_sym
+        proxy = @corresponding_proxies.detect do |_proxy|
+          _proxy.params.has_key?(param_key)
+        end
+        return nil unless proxy
+        _block, opts = proxy.params[param_key]
+        opts[:type]
+      end
+
+      def dump_value(k, v, indent, nindent)
+        if secret_param?(k)
+          "#{nindent}#{k} xxxxxx\n"
+        else
+          if @v1_config
+            case param_type(k)
+            when :string
+              "#{nindent}#{k} \"#{self.class.unescape_parameter(v)}\"\n"
+            when :enum, :integer, :float, :size, :bool, :time
+              "#{nindent}#{k} #{v}\n"
+            when :hash, :array
+              "#{nindent}#{k} #{v}\n"
+            else
+              # Unknown type
+              "#{nindent}#{k} #{v}\n"
+            end
+          else
+            "#{nindent}#{k} #{v}\n"
+          end
+        end
+      end
+
       def self.unescape_parameter(v)
         result = ''
         v.each_char { |c| result << LiteralParser.unescape_char(c) }

data/lib/fluent/config/error.rb

@@ -20,7 +20,4 @@ module Fluent
 
   class ConfigParseError < ConfigError
   end
-
-  class ObsoletedParameterError < ConfigError
-  end
 end

data/lib/fluent/config/literal_parser.rb

@@ -181,8 +181,6 @@ module Fluent
           "\f"
         when "b"
           "\b"
-        when "0"
-          "\0"
         when /[a-zA-Z0-9]/
           parse_error! "unexpected back-slash escape character '#{c}'"
         else  # symbols

data/lib/fluent/config/parser.rb

@@ -81,10 +81,10 @@ module Fluent
             pattern = path
           end
 
-          Dir.glob(pattern).sort.each { |path|
-            basepath = File.dirname(path)
-            fname = File.basename(path)
-            File.open(path) { |f|
+          Dir.glob(pattern).sort.each { |entry|
+            basepath = File.dirname(entry)
+            fname = File.basename(entry)
+            File.open(entry) { |f|
               Parser.new(basepath, f.each_line, fname).parse!(allow_include, nil, attrs, elems)
             }
           }

data/lib/fluent/config/section.rb

@@ -26,13 +26,26 @@ module Fluent
         'Fluent::Config::Section'
       end
 
-      def initialize(params = {})
+      def initialize(params = {}, config_element = nil)
         @klass = 'Fluent::Config::Section'
         @params = params
+        @corresponding_config_element = config_element
       end
 
       alias :object_id :__id__
 
+      def corresponding_config_element
+        @corresponding_config_element
+      end
+
+      def class
+        Section
+      end
+
+      def to_s
+        inspect
+      end
+
       def inspect
         "<Fluent::Config::Section #{@params.to_json}>"
       end
@@ -62,6 +75,19 @@ module Fluent
         @params[key.to_sym]
       end
 
+      def respond_to?(symbol, include_all=false)
+        case symbol
+        when :inspect, :nil?, :to_h, :+, :instance_of?, :kind_of?, :[], :respond_to?, :respond_to_missing?
+          true
+        when :!, :!= , :==, :equal?, :instance_eval, :instance_exec
+          true
+        when :method_missing, :singleton_method_added, :singleton_method_removed, :singleton_method_undefined
+          include_all
+        else
+          false
+        end
+      end
+
       def respond_to_missing?(symbol, include_private)
         @params.has_key?(symbol)
       end
@@ -70,7 +96,7 @@ module Fluent
         if @params.has_key?(name)
           @params[name]
         else
-          super
+          ::Kernel.raise ::NoMethodError, "undefined method `#{name}' for #{self.inspect}"
         end
       end
     end
@@ -88,7 +114,7 @@ module Fluent
 
         proxy.defaults.each_pair do |name, defval|
           varname = name.to_sym
-          section_params[varname] = defval
+          section_params[varname] = (defval.dup rescue defval)
         end
 
         if proxy.argument
@@ -97,10 +123,9 @@ module Fluent
             section_params[key] = self.instance_exec(conf.arg, opts, name, &block)
           end
           unless section_params.has_key?(proxy.argument.first)
-            logger.error "config error in:\n#{conf}"
+            logger.error "config error in:\n#{conf}" if logger # logger should exist, but somethimes it's nil (e.g, in tests)
             raise ConfigError, "'<#{proxy.name} ARG>' section requires argument" + section_stack
           end
-          # argument should NOT be deprecated... (argument always has a value: '')
         end
 
         proxy.params.each_pair do |name, defval|
@@ -113,23 +138,9 @@ module Fluent
                     conf[opts[:alias].to_s]
                   end
             section_params[varname] = self.instance_exec(val, opts, name, &block)
-
-            # Source of definitions of deprecated/obsoleted:
-            # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Deprecated_and_obsolete_features
-            #
-            # Deprecated: These deprecated features can still be used, but should be used with caution
-            #             because they are expected to be removed entirely sometime in the future.
-            # Obsoleted: These obsolete features have been entirely removed from JavaScript and can no longer be used.
-            if opts[:deprecated]
-              logger.warn "'#{name}' parameter is deprecated: #{opts[:deprecated]}"
-            end
-            if opts[:obsoleted]
-              logger.error "config error in:\n#{conf}" if logger
-              raise ObsoletedParameterError, "'#{name}' parameter is already removed: #{opts[:obsoleted]}" + section_stack
-            end
           end
           unless section_params.has_key?(varname)
-            logger.error "config error in:\n#{conf}"
+            logger.error "config error in:\n#{conf}" if logger
             raise ConfigError, "'#{name}' parameter is required" + section_stack
           end
         end
@@ -137,39 +148,39 @@ module Fluent
         check_unused_section(proxy, conf, plugin_class)
 
         proxy.sections.each do |name, subproxy|
-          varname = subproxy.param_name.to_sym
+          varname = subproxy.variable_name
           elements = (conf.respond_to?(:elements) ? conf.elements : []).select{ |e| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }
+          if elements.empty? && subproxy.init? && !subproxy.multi?
+            elements << Fluent::Config::Element.new(subproxy.name.to_s, '', {}, [])
+          end
+
           # set subproxy for secret option
           elements.each { |element|
             element.corresponding_proxies << subproxy
           }
 
           if subproxy.required? && elements.size < 1
-            logger.error "config error in:\n#{conf}"
+            logger.error "config error in:\n#{conf}" if logger
             raise ConfigError, "'<#{subproxy.name}>' sections are required" + section_stack
           end
           if subproxy.multi?
             section_params[varname] = elements.map{ |e| generate(subproxy, e, logger, plugin_class, stack + [subproxy.name]) }
           else
             if elements.size > 1
-              logger.error "config error in:\n#{conf}"
+              logger.error "config error in:\n#{conf}" if logger
               raise ConfigError, "'<#{subproxy.name}>' section cannot be written twice or more" + section_stack
             end
             section_params[varname] = generate(subproxy, elements.first, logger, plugin_class, stack + [subproxy.name])
           end
         end
 
-        Section.new(section_params)
+        Section.new(section_params, conf)
       end
 
       def self.check_unused_section(proxy, conf, plugin_class)
         elems = conf.respond_to?(:elements) ? conf.elements : []
         elems.each { |e|
           next if plugin_class.nil? && Fluent::Config::V1Parser::ELEM_SYMBOLS.include?(e.name) # skip pre-defined non-plugin elements because it doens't have proxy section
-          # In v0.12, buffer and output parameters are defined in same place.
-          # It causes same configuration is used in different buffer / output
-          # and buffer plugin's sections are always empty. It should be skipped.
-          next if proxy.sections.empty?
 
           unless proxy.sections.any? { |name, subproxy| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }
             parent_name = if conf.arg.empty?

data/lib/fluent/config/types.rb

@@ -17,7 +17,6 @@
 require 'json'
 
 require 'fluent/config/error'
-require 'fluent/configurable'
 
 module Fluent
   module Config
@@ -65,7 +64,7 @@ module Fluent
       end
     end
 
-    STRING_TYPE = Proc.new { |val, opts| val.to_s.force_encoding(Encoding::UTF_8) }
+    STRING_TYPE = Proc.new { |val, opts| val }
     ENUM_TYPE = Proc.new { |val, opts|
       s = val.to_sym
       list = opts[:list]
@@ -86,7 +85,7 @@ module Fluent
         value
       else
         case type
-        when :string  then value.to_s.force_encoding(Encoding::UTF_8)
+        when :string  then value.to_s
         when :integer then value.to_i
         when :float   then value.to_f
         when :size then Config.size_value(value)
@@ -134,14 +133,4 @@ module Fluent
       end
     }
   end
-
-  Configurable.register_type(:string,  Config::STRING_TYPE)
-  Configurable.register_type(:enum,    Config::ENUM_TYPE)
-  Configurable.register_type(:integer, Config::INTEGER_TYPE)
-  Configurable.register_type(:float,   Config::FLOAT_TYPE)
-  Configurable.register_type(:size,    Config::SIZE_TYPE)
-  Configurable.register_type(:bool,    Config::BOOL_TYPE)
-  Configurable.register_type(:time,    Config::TIME_TYPE)
-  Configurable.register_type(:hash,    Config::HASH_TYPE)
-  Configurable.register_type(:array,   Config::ARRAY_TYPE)
 end

data/lib/fluent/config/v1_parser.rb

@@ -148,9 +148,7 @@ module Fluent
 
       def eval_include(attrs, elems, uri)
         u = URI.parse(uri)
-        if u.scheme == 'file' || (!u.scheme.nil? && u.scheme.length == 1) || u.path == uri # file path
-          # When the Windows absolute path then u.scheme.length == 1
-          # e.g. C:
+        if u.scheme == 'file' || u.path == uri  # file path
           path = u.path
           if path[0] != ?/
             pattern = File.expand_path("#{@include_basepath}/#{path}")

data/lib/fluent/configurable.rb

@@ -19,7 +19,7 @@ require 'fluent/config/section'
 require 'fluent/config/error'
 require 'fluent/registry'
 require 'fluent/plugin'
-require 'fluent/mixin'
+require 'fluent/config/types'
 
 module Fluent
   module Configurable
@@ -28,19 +28,22 @@ module Fluent
     end
 
     def initialize
+      super
       # to simulate implicit 'attr_accessor' by config_param / config_section and its value by config_set_default
       proxy = self.class.merged_configure_proxy
       proxy.params.keys.each do |name|
+        next if name.to_s.start_with?('@')
         if proxy.defaults.has_key?(name)
           instance_variable_set("@#{name}".to_sym, proxy.defaults[name])
         end
       end
       proxy.sections.keys.each do |name|
+        next if name.to_s.start_with?('@')
         subproxy = proxy.sections[name]
         if subproxy.multi?
-          instance_variable_set("@#{subproxy.param_name}".to_sym, [])
+          instance_variable_set("@#{subproxy.variable_name}".to_sym, [])
         else
-          instance_variable_set("@#{subproxy.param_name}".to_sym, nil)
+          instance_variable_set("@#{subproxy.variable_name}".to_sym, nil)
         end
       end
     end
@@ -48,18 +51,27 @@ module Fluent
     def configure(conf)
       @config = conf
 
-      logger = self.respond_to?(:log) ? log : $log
+      logger = self.respond_to?(:log) ? log : (defined?($log) ? $log : nil)
       proxy = self.class.merged_configure_proxy
       conf.corresponding_proxies << proxy
 
+      if self.respond_to?(:owner) && self.owner
+        owner_proxy = owner.class.merged_configure_proxy
+        if proxy.configured_in_section
+          owner_proxy = owner_proxy.sections[proxy.configured_in_section]
+        end
+        proxy.overwrite_defaults(owner_proxy) if owner_proxy
+      end
+
       # In the nested section, can't get plugin class through proxies so get plugin class here
-      plugin_class = Fluent::Plugin.lookup_name_from_class(proxy.name.to_s)
+      plugin_class = Fluent::Plugin.lookup_type_from_class(proxy.name.to_s)
       root = Fluent::Config::SectionGenerator.generate(proxy, conf, logger, plugin_class)
       @config_root_section = root
 
       root.instance_eval{ @params.keys }.each do |param_name|
+        next if param_name.to_s.start_with?('@')
         varname = "@#{param_name}".to_sym
-        if (! root[param_name].nil?) || instance_variable_get(varname).nil?
+        if (! root[param_name].nil?) || (instance_variable_defined?(varname) && instance_variable_get(varname).nil?)
           instance_variable_set(varname, root[param_name])
         end
       end
@@ -82,6 +94,20 @@ module Fluent
       CONFIG_TYPE_REGISTRY.lookup(type)
     end
 
+    {
+      string: Config::STRING_TYPE,
+      enum: Config::ENUM_TYPE,
+      integer: Config::INTEGER_TYPE,
+      float: Config::FLOAT_TYPE,
+      size: Config::SIZE_TYPE,
+      bool: Config::BOOL_TYPE,
+      time: Config::TIME_TYPE,
+      hash: Config::HASH_TYPE,
+      array: Config::ARRAY_TYPE,
+    }.each do |name, type|
+      register_type(name, type)
+    end
+
     module ClassMethods
       def configure_proxy_map
         map = {}
@@ -92,15 +118,21 @@ module Fluent
       def configure_proxy(mod_name)
         map = configure_proxy_map
         unless map[mod_name]
-          proxy = Fluent::Config::ConfigureProxy.new(mod_name, required: true, multi: false)
+          type_lookup = ->(type) { Fluent::Configurable.lookup_type(type) }
+          proxy = Fluent::Config::ConfigureProxy.new(mod_name, required: true, multi: false, type_lookup: type_lookup)
           map[mod_name] = proxy
         end
         map[mod_name]
       end
 
-      def config_param(name, *args, &block)
-        configure_proxy(self.name).config_param(name, *args, &block)
-        attr_accessor name
+      def configured_in(section_name)
+        configure_proxy(self.name).configured_in(section_name)
+      end
+
+      def config_param(name, type = nil, **kwargs, &block)
+        configure_proxy(self.name).config_param(name, type, **kwargs, &block)
+        # reserved names '@foo' are invalid as attr_accessor name
+        attr_accessor(name) unless kwargs[:skip_accessor] || Fluent::Config::Element::RESERVED_PARAMETERS.include?(name.to_s)
       end
 
       def config_set_default(name, defval)
@@ -111,9 +143,12 @@ module Fluent
         configure_proxy(self.name).config_set_desc(name, desc)
       end
 
-      def config_section(name, *args, &block)
-        configure_proxy(self.name).config_section(name, *args, &block)
-        attr_accessor configure_proxy(self.name).sections[name].param_name
+      def config_section(name, **kwargs, &block)
+        configure_proxy(self.name).config_section(name, **kwargs, &block)
+        variable_name = configure_proxy(self.name).sections[name].variable_name
+        unless self.respond_to?(variable_name)
+          attr_accessor variable_name
+        end
       end
 
       def desc(description)
@@ -137,7 +172,4 @@ module Fluent
       end
     end
   end
-
-  # load default types
-  require 'fluent/config/types'
 end