fluentd 0.12.40 → 1.6.2

data/lib/fluent/config/configure_proxy.rb

@@ -14,13 +14,12 @@
 #    limitations under the License.
 #
 
-require 'fluent/configurable'
-
 module Fluent
   module Config
     class ConfigureProxy
-      attr_accessor :name, :final, :param_name, :required, :multi, :alias, :argument, :params, :defaults, :descriptions, :sections
-      # config_param :desc, :string, :default => '....'
+      attr_accessor :name, :final, :param_name, :init, :required, :multi, :alias, :configured_in_section
+      attr_accessor :argument, :params, :defaults, :descriptions, :sections
+      # config_param :desc, :string, default: '....'
       # config_set_default :buffer_type, :memory
       #
       # config_section :default, required: true, multi: false do
@@ -38,20 +37,48 @@ module Fluent
       #   end
       # end
 
-      def initialize(name, opts = {})
+      def initialize(name, root: false, param_name: nil, final: nil, init: nil, required: nil, multi: nil, alias: nil, type_lookup:)
         @name = name.to_sym
-        @final = opts.fetch(:final, false)
+        @final = final
+
+        # For ConfigureProxy of root section, "@name" should be a class name of plugins.
+        # Otherwise (like subsections), "@name" should be a name of section, like "buffer", "store".
+        # For subsections, name will be used as parameter names (unless param_name exists), so overriding proxy's name
+        #   should override "@name".
+        @root_section = root
+
+        @param_name = param_name && param_name.to_sym
+        @init = init
+        @required = required
+        @multi = multi
+        @alias = binding.local_variable_get(:alias)
+        @type_lookup = type_lookup
+
+        raise "init and required are exclusive" if @init && @required
 
-        @param_name = (opts[:param_name] || @name).to_sym
-        @required = opts[:required]
-        @multi = opts[:multi]
-        @alias = opts[:alias]
+        # specify section name for viewpoint of owner(parent) plugin
+        # for buffer plugins: all params are in <buffer> section of owner
+        # others: <storage>, <format> (formatter/parser), ...
+        @configured_in_section = nil
 
         @argument = nil # nil: ignore argument
         @params = {}
         @defaults = {}
         @descriptions = {}
         @sections = {}
+        @current_description = nil
+      end
+
+      def variable_name
+        @param_name || @name
+      end
+
+      def root?
+        @root_section
+      end
+
+      def init?
+        @init.nil? ? false : @init
       end
 
       def required?
@@ -63,18 +90,40 @@ module Fluent
       end
 
       def final?
-        @final
+        !!@final
       end
 
       def merge(other) # self is base class, other is subclass
         return merge_for_finalized(other) if self.final?
 
-        options = {
-          param_name: other.param_name,
-          required: (other.required.nil? ? self.required : other.required),
-          multi: (other.multi.nil? ? self.multi : other.multi)
-        }
-        merged = self.class.new(other.name, options)
+        [:param_name, :required, :multi, :alias, :configured_in_section].each do |prohibited_name|
+          if overwrite?(other, prohibited_name)
+            raise ConfigError, "BUG: subclass cannot overwrite base class's config_section: #{prohibited_name}"
+          end
+        end
+
+        options = {}
+        # param_name affects instance variable name, which is just "internal" of each plugins.
+        # so it must not be changed. base class's name (or param_name) is always used.
+        options[:param_name] = @param_name
+
+        # subclass cannot overwrite base class's definition
+        options[:init] = @init.nil? ? other.init : self.init
+        options[:required] = @required.nil? ? other.required : self.required
+        options[:multi] = @multi.nil? ? other.multi : self.multi
+        options[:alias] = @alias.nil? ? other.alias : self.alias
+        options[:final] = @final || other.final
+        options[:type_lookup] = @type_lookup
+
+        merged = if self.root?
+                   options[:root] = true
+                   self.class.new(other.name, options)
+                 else
+                   self.class.new(@name, options)
+                 end
+
+        # configured_in MUST be kept
+        merged.configured_in_section = self.configured_in_section || other.configured_in_section
 
         merged.argument = other.argument || self.argument
         merged.params = self.params.merge(other.params)
@@ -98,20 +147,41 @@ module Fluent
 
       def merge_for_finalized(other)
         # list what subclass can do for finalized section
-        #  * overwrite param_name to escape duplicated name of instance variable
         #  * append params/defaults/sections which are missing in superclass
+        #  * change default values of superclass
+        #  * overwrite init to make it enable to instantiate section objects with added default values
+
+        if other.final == false && overwrite?(other, :final)
+          raise ConfigError, "BUG: subclass cannot overwrite finalized base class's config_section"
+        end
+
+        [:param_name, :required, :multi, :alias, :configured_in_section].each do |prohibited_name|
+          if overwrite?(other, prohibited_name)
+            raise ConfigError, "BUG: subclass cannot overwrite base class's config_section: #{prohibited_name}"
+          end
+        end
 
-        options = {
-          param_name: other.param_name,
-          required: (self.required.nil? ? other.required : self.required),
-          multi: (self.multi.nil? ? other.multi : self.multi),
-          final: true,
-        }
-        merged = self.class.new(other.name, options)
+        options = {}
+        options[:param_name] = @param_name
+        options[:init] = @init || other.init
+        options[:required] = @required.nil? ? other.required : self.required
+        options[:multi] = @multi.nil? ? other.multi : self.multi
+        options[:alias] = @alias.nil? ? other.alias : self.alias
+        options[:final]  = true
+        options[:type_lookup] = @type_lookup
+
+        merged = if self.root?
+                   options[:root] = true
+                   self.class.new(other.name, options)
+                 else
+                   self.class.new(@name, options)
+                 end
+
+        merged.configured_in_section = self.configured_in_section || other.configured_in_section
 
         merged.argument = self.argument || other.argument
         merged.params = other.params.merge(self.params)
-        merged.defaults = other.defaults.merge(self.defaults)
+        merged.defaults = self.defaults.merge(other.defaults)
         merged.sections = {}
         (self.sections.keys + other.sections.keys).uniq.each do |section_key|
           self_section = self.sections[section_key]
@@ -129,49 +199,88 @@ module Fluent
         merged
       end
 
-      def option_value_type!(name, opts, key, klass)
-        if opts.has_key?(key) && !opts[key].is_a?(klass)
-          raise ArgumentError, "#{name}: #{key} must be a #{klass}, but #{opts[key].class}"
+      def overwrite_defaults(other) # other is owner plugin's corresponding proxy
+        self.defaults = self.defaults.merge(other.defaults)
+        self.sections.keys.each do |section_key|
+          if other.sections.has_key?(section_key)
+            self.sections[section_key].overwrite_defaults(other.sections[section_key])
+          end
         end
       end
 
-      def parameter_configuration(name, *args, &block)
-        name = name.to_sym
+      def option_value_type!(name, opts, key, klass=nil, type: nil)
+        if opts.has_key?(key)
+          if klass && !opts[key].is_a?(klass)
+            raise ArgumentError, "#{name}: #{key} must be a #{klass}, but #{opts[key].class}"
+          end
+          case type
+          when :boolean
+            unless opts[key].is_a?(TrueClass) || opts[key].is_a?(FalseClass)
+              raise ArgumentError, "#{name}: #{key} must be true or false, but #{opts[key].class}"
+            end
+          when nil
+            # ignore
+          else
+            raise "unknown type: #{type} for option #{key}"
+          end
+        end
+      end
 
-        opts = {}
-        args.each { |a|
-          if a.is_a?(Symbol)
-            opts[:type] = a
-          elsif a.is_a?(Hash)
-            opts.merge!(a)
+      def config_parameter_option_validate!(name, type, **kwargs, &block)
+        if type.nil? && !block
+          type = :string
+        end
+        kwargs.each_key do |key|
+          case key
+          when :default, :alias, :secret, :skip_accessor, :deprecated, :obsoleted, :desc
+            # valid for all types
+          when :list
+            raise ArgumentError, ":list is valid only for :enum type, but #{type}: #{name}" if type != :enum
+          when :value_type
+            raise ArgumentError, ":value_type is valid only for :hash and :array, but #{type}: #{name}" if type != :hash && type != :array
+          when :symbolize_keys
+            raise ArgumentError, ":symbolize_keys is valid only for :hash, but #{type}: #{name}" if type != :hash
           else
-            raise ArgumentError, "#{self.name}: wrong number of arguments (#{1 + args.length} for #{block ? 2 : 3})"
+            raise ArgumentError, "unknown option '#{key}' for configuration parameter: #{name}"
           end
-        }
+        end
+      end
+
+      def parameter_configuration(name, type = nil, **kwargs, &block)
+        config_parameter_option_validate!(name, type, **kwargs, &block)
+
+        name = name.to_sym
 
-        type = opts[:type]
         if block && type
           raise ArgumentError, "#{name}: both of block and type cannot be specified"
+        elsif !block && !type
+          type = :string
         end
+        opts = {}
+        opts[:type] = type
+        opts.merge!(kwargs)
 
         begin
-          type = :string if type.nil?
-          block ||= Configurable.lookup_type(type)
+          block ||= @type_lookup.call(type)
         rescue ConfigError
           # override error message
           raise ArgumentError, "#{name}: unknown config_argument type `#{type}'"
         end
 
+        # options for config_param
         option_value_type!(name, opts, :desc, String)
         option_value_type!(name, opts, :alias, Symbol)
+        option_value_type!(name, opts, :secret, type: :boolean)
         option_value_type!(name, opts, :deprecated, String)
         option_value_type!(name, opts, :obsoleted, String)
         if type == :enum
-          if !opts.has_key?(:list) || !opts[:list].all?{|v| v.is_a?(Symbol) }
+          if !opts.has_key?(:list) || !opts[:list].is_a?(Array) || opts[:list].empty? || !opts[:list].all?{|v| v.is_a?(Symbol) }
             raise ArgumentError, "#{name}: enum parameter requires :list of Symbols"
           end
         end
+        option_value_type!(name, opts, :symbolize_keys, type: :boolean)
         option_value_type!(name, opts, :value_type, Symbol) # hash, array
+        option_value_type!(name, opts, :skip_accessor, type: :boolean)
 
         if opts.has_key?(:default)
           config_set_default(name, opts[:default])
@@ -188,18 +297,25 @@ module Fluent
         [name, block, opts]
       end
 
-      def config_argument(name, *args, &block)
+      def configured_in(section_name)
+        if @configured_in_section
+          raise ArgumentError, "#{self.name}: configured_in called twice"
+        end
+        @configured_in_section = section_name.to_sym
+      end
+
+      def config_argument(name, type = nil, **kwargs, &block)
         if @argument
           raise ArgumentError, "#{self.name}: config_argument called twice"
         end
-        name, block, opts = parameter_configuration(name, *args, &block)
+        name, block, opts = parameter_configuration(name, type, **kwargs, &block)
 
         @argument = [name, block, opts]
         name
       end
 
-      def config_param(name, *args, &block)
-        name, block, opts = parameter_configuration(name, *args, &block)
+      def config_param(name, type = nil, **kwargs, &block)
+        name, block, opts = parameter_configuration(name, type, **kwargs, &block)
 
         if @current_description
           config_set_desc(name, @current_description)
@@ -237,18 +353,13 @@ module Fluent
         @current_description = description
       end
 
-      def config_section(name, *args, &block)
+      def config_section(name, **kwargs, &block)
         unless block_given?
           raise ArgumentError, "#{name}: config_section requires block parameter"
         end
         name = name.to_sym
 
-        opts = {}
-        unless args.empty? || args.size == 1 && args.first.is_a?(Hash)
-          raise ArgumentError, "#{self.name}: unknown config_section arguments: #{args.inspect}"
-        end
-
-        sub_proxy = ConfigureProxy.new(name, *args)
+        sub_proxy = ConfigureProxy.new(name, type_lookup: @type_lookup, **kwargs)
         sub_proxy.instance_exec(&block)
 
         @params.delete(name)
@@ -257,20 +368,57 @@ module Fluent
         name
       end
 
-      def dump(level = 0)
-        dumped_config = ""
-        indent = " " * level
+      def dump_config_definition
+        dumped_config = {}
+        if @argument
+          argument_name, _block, options = @argument
+          options[:required] = !@defaults.key?(argument_name)
+          options[:argument] = true
+          dumped_config[argument_name] = options
+        end
         @params.each do |name, config|
-          dumped_config << "#{indent}#{name}: #{config[1][:type]}: <#{@defaults[name].inspect}>"
-          dumped_config << " # #{@descriptions[name]}" if @descriptions[name]
-          dumped_config << "\n"
+          dumped_config[name] = config[1]
+          dumped_config[name][:required] = !@defaults.key?(name)
+          dumped_config[name][:default] = @defaults[name] if @defaults.key?(name)
+          dumped_config[name][:desc] = @descriptions[name] if @descriptions.key?(name)
+        end
+        # Overwrite by config_set_default
+        @defaults.each do |name, value|
+          if @params.key?(name) || (@argument && @argument.first == name)
+            dumped_config[name][:default] = value
+          else
+            dumped_config[name] = { default: value }
+          end
+        end
+        # Overwrite by config_set_desc
+        @descriptions.each do |name, value|
+          if @params.key?(name)
+            dumped_config[name][:desc] = value
+          else
+            dumped_config[name] = { desc: value }
+          end
         end
         @sections.each do |section_name, sub_proxy|
-          dumped_config << "#{indent}#{section_name}\n#{sub_proxy.dump(level + 1)}"
+          if dumped_config.key?(section_name)
+            dumped_config[section_name].update(sub_proxy.dump_config_definition)
+          else
+            dumped_config[section_name] = sub_proxy.dump_config_definition
+            dumped_config[section_name][:required] = sub_proxy.required?
+            dumped_config[section_name][:multi] = sub_proxy.multi?
+            dumped_config[section_name][:alias] = sub_proxy.alias
+            dumped_config[section_name][:section] = true
+          end
         end
         dumped_config
       end
+
+      private
+
+      def overwrite?(other, attribute_name)
+        value = instance_variable_get("@#{attribute_name}")
+        other_value = other.__send__(attribute_name)
+        !value.nil? && !other_value.nil? && value != other_value
+      end
     end
   end
 end
-

data/lib/fluent/config/dsl.rb

@@ -22,6 +22,8 @@ require 'fluent/config/element'
 module Fluent
   module Config
     module DSL
+      RESERVED_PARAMETERS = [:type, :id, :log_level, :label] # Need '@' prefix for reserved parameters
+
       module Parser
         def self.read(path)
           path = File.expand_path(path)
@@ -82,6 +84,10 @@ module Fluent
           @proxy    = proxy
         end
 
+        def to_int
+          __id__
+        end
+
         def method_missing(name, *args, &block)
           ::Kernel.raise ::ArgumentError, "Configuration DSL Syntax Error: only one argument allowed" if args.size > 1
           value = args.first
@@ -91,11 +97,12 @@ module Fluent
             proxy.element.instance_exec(&block)
             @elements.push(proxy.to_config_element)
           else
-            @attrs[name.to_s] = if value.is_a?(Array) || value.is_a?(Hash)
-                                  JSON.dump(value)
-                                else
-                                  value.to_s
-                                end
+            param_name = RESERVED_PARAMETERS.include?(name) ? "@#{name}" : name.to_s
+            @attrs[param_name] = if value.is_a?(Array) || value.is_a?(Hash)
+                                   JSON.dump(value)
+                                 else
+                                   value.to_s
+                                 end
           end
 
           self

data/lib/fluent/config/element.rb

@@ -32,11 +32,39 @@ module Fluent
         @v1_config = false
         @corresponding_proxies = [] # some plugins use flat parameters, e.g. in_http doesn't provide <format> section for parser.
         @unused_in = false # if this element is not used in plugins, correspoing plugin name and parent element name is set, e.g. [source, plugin class].
+
+        # it's global logger, not plugin logger: deprecated message should be global warning, not plugin level.
+        @logger = defined?($log) ? $log : nil
+
+        @target_worker_ids = []
       end
 
-      attr_accessor :name, :arg, :elements, :unused, :v1_config, :corresponding_proxies, :unused_in
+      attr_accessor :name, :arg, :unused, :v1_config, :corresponding_proxies, :unused_in
+      attr_writer :elements
+      attr_reader :target_worker_ids
+
+      RESERVED_PARAMETERS_COMPAT = {
+        '@type' => 'type',
+        '@id' => 'id',
+        '@log_level' => 'log_level',
+        '@label' => nil,
+      }
+      RESERVED_PARAMETERS = RESERVED_PARAMETERS_COMPAT.keys
+
+      def elements(*names, name: nil, arg: nil)
+        raise ArgumentError, "name and names are exclusive" if name && !names.empty?
+        raise ArgumentError, "arg is available only with name" if arg && !name
+
+        if name
+          @elements.select{|e| e.name == name && (!arg || e.arg == arg) }
+        elsif !names.empty?
+          @elements.select{|e| names.include?(e.name) }
+        else
+          @elements
+        end
+      end
 
-      def add_element(name, arg='')
+      def add_element(name, arg = '')
         e = Element.new(name, arg, {}, [])
         e.v1_config = @v1_config
         @elements << e
@@ -48,6 +76,11 @@ module Fluent
         "name:#{@name}, arg:#{@arg}, " + attrs + ", " + @elements.inspect
       end
 
+      # Used by PP and Pry
+      def pretty_print(q)
+        q.text(inspect)
+      end
+
       # This method assumes _o_ is an Element object. Should return false for nil or other object
       def ==(o)
         self.name == o.name && self.arg == o.arg &&
@@ -63,6 +96,7 @@ module Fluent
         e
       end
 
+      # no code in fluentd uses this method
       def each_element(*names, &block)
         if names.empty?
           @elements.each(&block)
@@ -76,7 +110,7 @@ module Fluent
       end
 
       def has_key?(key)
-        @unused_in = false # some sections, e.g. <store> in copy, is not defined by config_section so clear unused flag for better warning message in chgeck_not_fetched.
+        @unused_in = false # some sections, e.g. <store> in copy, is not defined by config_section so clear unused flag for better warning message in check_not_fetched.
         @unused.delete(key)
         super
       end
@@ -84,6 +118,12 @@ module Fluent
       def [](key)
         @unused_in = false # ditto
         @unused.delete(key)
+
+        if RESERVED_PARAMETERS.include?(key) && !has_key?(key) && has_key?(RESERVED_PARAMETERS_COMPAT[key])
+          @logger.warn "'#{RESERVED_PARAMETERS_COMPAT[key]}' is deprecated parameter name. use '#{key}' instead." if @logger
+          return self[RESERVED_PARAMETERS_COMPAT[key]]
+        end
+
         super
       end
 
@@ -108,11 +148,7 @@ module Fluent
           out << "#{indent}<#{@name} #{@arg}>\n"
         end
         each_pair { |k, v|
-          if secret_param?(k)
-            out << "#{nindent}#{k} xxxxxx\n"
-          else
-            out << "#{nindent}#{k} #{v}\n"
-          end
+          out << dump_value(k, v, nindent)
         }
         @elements.each { |e|
           out << e.to_s(nest + 1)
@@ -124,6 +160,8 @@ module Fluent
       def to_masked_element
         new_elems = @elements.map { |e| e.to_masked_element }
         new_elem = Element.new(@name, @arg, {}, new_elems, @unused)
+        new_elem.v1_config = @v1_config
+        new_elem.corresponding_proxies = @corresponding_proxies
         each_pair { |k, v|
           new_elem[k] = secret_param?(k) ? 'xxxxxx' : v
         }
@@ -135,7 +173,7 @@ module Fluent
 
         param_key = key.to_sym
         @corresponding_proxies.each { |proxy|
-          block, opts = proxy.params[param_key]
+          _block, opts = proxy.params[param_key]
           if opts && opts.has_key?(:secret)
             return opts[:secret]
           end
@@ -144,11 +182,71 @@ module Fluent
         false
       end
 
+      def param_type(key)
+        return nil if @corresponding_proxies.empty?
+
+        param_key = key.to_sym
+        proxy = @corresponding_proxies.detect do |_proxy|
+          _proxy.params.has_key?(param_key)
+        end
+        return nil unless proxy
+        _block, opts = proxy.params[param_key]
+        opts[:type]
+      end
+
+      def dump_value(k, v, nindent)
+        if secret_param?(k)
+          "#{nindent}#{k} xxxxxx\n"
+        else
+          if @v1_config
+            case param_type(k)
+            when :string
+              "#{nindent}#{k} \"#{self.class.unescape_parameter(v)}\"\n"
+            when :enum, :integer, :float, :size, :bool, :time
+              "#{nindent}#{k} #{v}\n"
+            when :hash, :array
+              "#{nindent}#{k} #{v}\n"
+            else
+              # Unknown type
+              "#{nindent}#{k} #{v}\n"
+            end
+          else
+            "#{nindent}#{k} #{v}\n"
+          end
+        end
+      end
+
       def self.unescape_parameter(v)
         result = ''
         v.each_char { |c| result << LiteralParser.unescape_char(c) }
         result
       end
+
+      def set_target_worker_id(worker_id)
+        @target_worker_ids = [worker_id]
+        @elements.each { |e|
+          e.set_target_worker_id(worker_id)
+        }
+      end
+
+      def set_target_worker_ids(worker_ids)
+        @target_worker_ids = worker_ids.uniq
+        @elements.each { |e|
+          e.set_target_worker_ids(worker_ids.uniq)
+        }
+      end
+
+      def for_every_workers?
+        @target_worker_ids.empty?
+      end
+
+      def for_this_worker?
+        @target_worker_ids.include?(Fluent::Engine.worker_id)
+      end
+
+      def for_another_worker?
+        !@target_worker_ids.empty? && !@target_worker_ids.include?(Fluent::Engine.worker_id)
+      end
     end
   end
 end

data/lib/fluent/config/literal_parser.rb

@@ -18,6 +18,7 @@ require 'stringio'
 
 require 'json'
 require 'yajl'
+require 'socket'
 require 'irb/ruby-lex'  # RubyLex
 
 require 'fluent/config/basic_parser'
@@ -162,6 +163,11 @@ module Fluent
         if @eval_context.nil?
           parse_error! "embedded code is not allowed in this file"
         end
+        # Add hostname and worker_id to code for preventing unused warnings
+        code = <<EOM + code
+hostname = Socket.gethostname
+worker_id = ENV['SERVERENGINE_WORKER_ID'] || ''
+EOM
         @eval_context.instance_eval(code)
       end
 
@@ -192,7 +198,7 @@ module Fluent
 
       def scan_json(is_array)
         result = nil
-        # Yajl does not raise ParseError for imcomplete json string, like '[1', '{"h"', '{"h":' or '{"h1":1'
+        # Yajl does not raise ParseError for incomplete json string, like '[1', '{"h"', '{"h":' or '{"h1":1'
         # This is the reason to use JSON module.
 
         buffer = (is_array ? "[" : "{")
@@ -209,7 +215,7 @@ module Fluent
             parsed = nil
             begin
               parsed = JSON.parse(buffer + line_buffer.rstrip.sub(/,$/, '') + (is_array ? "]" : "}"))
-            rescue JSON::ParserError => e
+            rescue JSON::ParserError
               # This '#' is in json string literals
             end
 
@@ -237,7 +243,7 @@ module Fluent
           line_buffer << char
           begin
             result = JSON.parse(buffer + line_buffer)
-          rescue JSON::ParserError => e
+          rescue JSON::ParserError
             # Incomplete json string yet
           end
         end

data/lib/fluent/config/parser.rb

@@ -81,10 +81,10 @@ module Fluent
             pattern = path
           end
 
-          Dir.glob(pattern).sort.each { |path|
-            basepath = File.dirname(path)
-            fname = File.basename(path)
-            File.open(path) { |f|
+          Dir.glob(pattern).sort.each { |entry|
+            basepath = File.dirname(entry)
+            fname = File.basename(entry)
+            File.open(entry) { |f|
               Parser.new(basepath, f.each_line, fname).parse!(allow_include, nil, attrs, elems)
             }
           }