fluentd 0.12.40 → 1.6.2

data/lib/fluent/config/section.rb

@@ -26,13 +26,26 @@ module Fluent
         'Fluent::Config::Section'
       end
 
-      def initialize(params = {})
+      def initialize(params = {}, config_element = nil)
         @klass = 'Fluent::Config::Section'
         @params = params
+        @corresponding_config_element = config_element
       end
 
       alias :object_id :__id__
 
+      def corresponding_config_element
+        @corresponding_config_element
+      end
+
+      def class
+        Section
+      end
+
+      def to_s
+        inspect
+      end
+
       def inspect
         "<Fluent::Config::Section #{@params.to_json}>"
       end
@@ -62,6 +75,23 @@ module Fluent
         @params[key.to_sym]
       end
 
+      def []=(key, value)
+        @params[key.to_sym] = value
+      end
+
+      def respond_to?(symbol, include_all=false)
+        case symbol
+        when :inspect, :nil?, :to_h, :+, :instance_of?, :kind_of?, :[], :respond_to?, :respond_to_missing?
+          true
+        when :!, :!= , :==, :equal?, :instance_eval, :instance_exec
+          true
+        when :method_missing, :singleton_method_added, :singleton_method_removed, :singleton_method_undefined
+          include_all
+        else
+          false
+        end
+      end
+
       def respond_to_missing?(symbol, include_private)
         @params.has_key?(symbol)
       end
@@ -70,7 +100,7 @@ module Fluent
         if @params.has_key?(name)
           @params[name]
         else
-          super
+          ::Kernel.raise ::NoMethodError, "undefined method `#{name}' for #{self.inspect}"
         end
       end
     end
@@ -88,7 +118,7 @@ module Fluent
 
         proxy.defaults.each_pair do |name, defval|
           varname = name.to_sym
-          section_params[varname] = defval
+          section_params[varname] = (defval.dup rescue defval)
         end
 
         if proxy.argument
@@ -97,7 +127,7 @@ module Fluent
             section_params[key] = self.instance_exec(conf.arg, opts, name, &block)
           end
           unless section_params.has_key?(proxy.argument.first)
-            logger.error "config error in:\n#{conf}"
+            logger.error "config error in:\n#{conf}" if logger # logger should exist, but somethimes it's nil (e.g, in tests)
             raise ConfigError, "'<#{proxy.name} ARG>' section requires argument" + section_stack
           end
           # argument should NOT be deprecated... (argument always has a value: '')
@@ -121,7 +151,7 @@ module Fluent
             #             because they are expected to be removed entirely sometime in the future.
             # Obsoleted: These obsolete features have been entirely removed from JavaScript and can no longer be used.
             if opts[:deprecated]
-              logger.warn "'#{name}' parameter is deprecated: #{opts[:deprecated]}"
+              logger.warn "'#{name}' parameter is deprecated: #{opts[:deprecated]}" if logger
             end
             if opts[:obsoleted]
               logger.error "config error in:\n#{conf}" if logger
@@ -129,7 +159,7 @@ module Fluent
             end
           end
           unless section_params.has_key?(varname)
-            logger.error "config error in:\n#{conf}"
+            logger.error "config error in:\n#{conf}" if logger
             raise ConfigError, "'#{name}' parameter is required" + section_stack
           end
         end
@@ -137,39 +167,46 @@ module Fluent
         check_unused_section(proxy, conf, plugin_class)
 
         proxy.sections.each do |name, subproxy|
-          varname = subproxy.param_name.to_sym
+          varname = subproxy.variable_name
           elements = (conf.respond_to?(:elements) ? conf.elements : []).select{ |e| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }
+          if elements.empty? && subproxy.init?
+            if subproxy.argument && !subproxy.defaults.has_key?(subproxy.argument.first)
+              raise ArgumentError, "#{name}: init is specified, but default value of argument is missing"
+            end
+            missing_keys = subproxy.params.keys.select{|param_name| !subproxy.defaults.has_key?(param_name)}
+            if !missing_keys.empty?
+              raise ArgumentError, "#{name}: init is specified, but there're parameters without default values:#{missing_keys.join(',')}"
+            end
+            elements << Fluent::Config::Element.new(subproxy.name.to_s, '', {}, [])
+          end
+
           # set subproxy for secret option
           elements.each { |element|
             element.corresponding_proxies << subproxy
           }
 
           if subproxy.required? && elements.size < 1
-            logger.error "config error in:\n#{conf}"
+            logger.error "config error in:\n#{conf}" if logger
             raise ConfigError, "'<#{subproxy.name}>' sections are required" + section_stack
           end
           if subproxy.multi?
             section_params[varname] = elements.map{ |e| generate(subproxy, e, logger, plugin_class, stack + [subproxy.name]) }
           else
             if elements.size > 1
-              logger.error "config error in:\n#{conf}"
+              logger.error "config error in:\n#{conf}" if logger
               raise ConfigError, "'<#{subproxy.name}>' section cannot be written twice or more" + section_stack
             end
             section_params[varname] = generate(subproxy, elements.first, logger, plugin_class, stack + [subproxy.name])
           end
         end
 
-        Section.new(section_params)
+        Section.new(section_params, conf)
       end
 
       def self.check_unused_section(proxy, conf, plugin_class)
         elems = conf.respond_to?(:elements) ? conf.elements : []
         elems.each { |e|
           next if plugin_class.nil? && Fluent::Config::V1Parser::ELEM_SYMBOLS.include?(e.name) # skip pre-defined non-plugin elements because it doens't have proxy section
-          # In v0.12, buffer and output parameters are defined in same place.
-          # It causes same configuration is used in different buffer / output
-          # and buffer plugin's sections are always empty. It should be skipped.
-          next if proxy.sections.empty?
 
           unless proxy.sections.any? { |name, subproxy| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }
             parent_name = if conf.arg.empty?

data/lib/fluent/config/types.rb

@@ -17,7 +17,6 @@
 require 'json'
 
 require 'fluent/config/error'
-require 'fluent/configurable'
 
 module Fluent
   module Config
@@ -61,11 +60,35 @@ module Fluent
       when ''
         true
       else
-        nil
+        # Current parser passes comment without actual values, e.g. "param #foo".
+        # parser should pass empty string in this case but changing behaviour may break existing environment so keep parser behaviour. Just ignore comment value in boolean handling for now.
+        if str.respond_to?('start_with?') && str.start_with?('#')
+          true
+        else
+          nil
+        end
       end
     end
 
-    STRING_TYPE = Proc.new { |val, opts| val.to_s.force_encoding(Encoding::UTF_8) }
+    def self.regexp_value(str)
+      return nil unless str
+      return Regexp.compile(str) unless str.start_with?("/")
+      right_slash_position = str.rindex("/")
+      if right_slash_position < str.size - 3
+        raise Fluent::ConfigError, "invalid regexp: missing right slash: #{str}"
+      end
+      options = str[(right_slash_position + 1)..-1]
+      option = 0
+      option |= Regexp::IGNORECASE if options.include?("i")
+      option |= Regexp::MULTILINE if options.include?("m")
+      Regexp.compile(str[1...right_slash_position], option)
+    end
+
+    STRING_TYPE = Proc.new { |val, opts|
+      v = val.to_s
+      v = v.frozen? ? v.dup : v # config_param can't assume incoming string is mutable
+      v.force_encoding(Encoding::UTF_8)
+    }
     ENUM_TYPE = Proc.new { |val, opts|
       s = val.to_sym
       list = opts[:list]
@@ -80,6 +103,7 @@ module Fluent
     SIZE_TYPE = Proc.new { |val, opts| Config.size_value(val) }
     BOOL_TYPE = Proc.new { |val, opts| Config.bool_value(val) }
     TIME_TYPE = Proc.new { |val, opts| Config.time_value(val) }
+    REGEXP_TYPE = Proc.new { |val, opts| Config.regexp_value(val) }
 
     REFORMAT_VALUE = ->(type, value) {
       if value.nil?
@@ -92,6 +116,7 @@ module Fluent
         when :size then Config.size_value(value)
         when :bool then Config.bool_value(value)
         when :time then Config.time_value(value)
+        when :regexp then Config.regexp_value(value)
         else
           raise "unknown type in REFORMAT: #{type}"
         end
@@ -134,14 +159,4 @@ module Fluent
       end
     }
   end
-
-  Configurable.register_type(:string,  Config::STRING_TYPE)
-  Configurable.register_type(:enum,    Config::ENUM_TYPE)
-  Configurable.register_type(:integer, Config::INTEGER_TYPE)
-  Configurable.register_type(:float,   Config::FLOAT_TYPE)
-  Configurable.register_type(:size,    Config::SIZE_TYPE)
-  Configurable.register_type(:bool,    Config::BOOL_TYPE)
-  Configurable.register_type(:time,    Config::TIME_TYPE)
-  Configurable.register_type(:hash,    Config::HASH_TYPE)
-  Configurable.register_type(:array,   Config::ARRAY_TYPE)
 end

data/lib/fluent/config/v1_parser.rb

@@ -82,7 +82,7 @@ module Fluent
           elsif skip(/\</)
             e_name = scan(ELEMENT_NAME)
             spacing
-            e_arg = scan_nonquoted_string(/(?:#{ZERO_OR_MORE_SPACING}\>)/)
+            e_arg = scan_string(/(?:#{ZERO_OR_MORE_SPACING}\>)/)
             spacing
             unless skip(/\>/)
               parse_error! "expected '>'"
@@ -106,7 +106,7 @@ module Fluent
           else
             k = scan_string(SPACING)
             spacing_without_comment
-            if prev_match.include?("\n") # support 'tag_mapped' like "without value" configuration
+            if prev_match.include?("\n") || eof? # support 'tag_mapped' like "without value" configuration
               attrs[k] = ""
             else
               if k == '@include'
@@ -157,7 +157,6 @@ module Fluent
           else
             pattern = path
           end
-
           Dir.glob(pattern).sort.each { |entry|
             basepath = File.dirname(entry)
             fname = File.basename(entry)
@@ -175,9 +174,8 @@ module Fluent
           ss = StringScanner.new(data)
           V1Parser.new(ss, basepath, fname, @eval_context).parse_element(true, nil, attrs, elems)
         end
-
       rescue SystemCallError => e
-        cpe = ConfigParseError.new("include error #{uri}")
+        cpe = ConfigParseError.new("include error #{uri} - #{e}")
         cpe.set_backtrace(e.backtrace)
         raise cpe
       end

data/lib/fluent/config.rb

@@ -20,18 +20,32 @@ require 'fluent/configurable'
 
 module Fluent
   module Config
-    def self.parse(str, fname, basepath = Dir.pwd, v1_config = false)
-      if fname =~ /\.rb$/
+    def self.parse(str, fname, basepath = Dir.pwd, v1_config = nil, syntax: :v1)
+      parser = if fname =~ /\.rb$/ || syntax == :ruby
+                 :ruby
+               elsif v1_config.nil?
+                 case syntax
+                 when :v1 then :v1
+                 when :v0 then :v0
+                 else
+                   raise ArgumentError, "Unknown Fluentd configuration syntax: '#{syntax}'"
+                 end
+               elsif v1_config then :v1
+               else :v0
+               end
+      case parser
+      when :v1
+        require 'fluent/config/v1_parser'
+        V1Parser.parse(str, fname, basepath, Kernel.binding)
+      when :v0
+        # TODO: show deprecated message in v1
+        require 'fluent/config/parser'
+        Parser.parse(str, fname, basepath)
+      when :ruby
         require 'fluent/config/dsl'
         Config::DSL::Parser.parse(str, File.join(basepath, fname))
       else
-        if v1_config
-          require 'fluent/config/v1_parser'
-          V1Parser.parse(str, fname, basepath, Kernel.binding)
-        else
-          require 'fluent/config/parser'
-          Parser.parse(str, fname, basepath)
-        end
+        raise "[BUG] unknown configuration parser specification:'#{parser}'"
       end
     end
 
@@ -39,15 +53,4 @@ module Fluent
       Element.new(name, '', {}, [])
     end
   end
-
-  module PluginId
-    def configure(conf)
-      @id = conf['@id'] || conf['id']
-      super
-    end
-
-    def plugin_id
-      @id ? @id : "object:#{object_id.to_s(16)}"
-    end
-  end
 end

data/lib/fluent/configurable.rb

@@ -19,7 +19,7 @@ require 'fluent/config/section'
 require 'fluent/config/error'
 require 'fluent/registry'
 require 'fluent/plugin'
-require 'fluent/mixin'
+require 'fluent/config/types'
 
 module Fluent
   module Configurable
@@ -28,40 +28,76 @@ module Fluent
     end
 
     def initialize
+      super
       # to simulate implicit 'attr_accessor' by config_param / config_section and its value by config_set_default
       proxy = self.class.merged_configure_proxy
       proxy.params.keys.each do |name|
+        next if name.to_s.start_with?('@')
         if proxy.defaults.has_key?(name)
           instance_variable_set("@#{name}".to_sym, proxy.defaults[name])
         end
       end
       proxy.sections.keys.each do |name|
+        next if name.to_s.start_with?('@')
         subproxy = proxy.sections[name]
         if subproxy.multi?
-          instance_variable_set("@#{subproxy.param_name}".to_sym, [])
+          instance_variable_set("@#{subproxy.variable_name}".to_sym, [])
         else
-          instance_variable_set("@#{subproxy.param_name}".to_sym, nil)
+          instance_variable_set("@#{subproxy.variable_name}".to_sym, nil)
         end
       end
     end
 
+    def configure_proxy_generate
+      proxy = self.class.merged_configure_proxy
+
+      if self.respond_to?(:owner) && self.owner
+        owner_proxy = owner.class.merged_configure_proxy
+        if proxy.configured_in_section
+          owner_proxy = owner_proxy.sections[proxy.configured_in_section]
+        end
+        proxy.overwrite_defaults(owner_proxy) if owner_proxy
+      end
+
+      proxy
+    end
+
+    def configured_section_create(name, conf = nil)
+      conf ||= Fluent::Config::Element.new(name.to_s, '', {}, [])
+      root_proxy = configure_proxy_generate
+      proxy = if name.nil? # root
+                root_proxy
+              else
+                root_proxy.sections[name]
+              end
+      # take care to raise Fluent::ConfigError if conf mismatched to proxy
+      Fluent::Config::SectionGenerator.generate(proxy, conf, nil, nil)
+    end
+
     def configure(conf)
       @config = conf
 
-      logger = self.respond_to?(:log) ? log : $log
-      proxy = self.class.merged_configure_proxy
+      logger = if self.respond_to?(:log)
+                 self.log
+               elsif self.respond_to?(:owner) && self.owner.respond_to?(:log)
+                 self.owner.log
+               elsif defined?($log)
+                 $log
+               else
+                 nil
+               end
+      proxy = configure_proxy_generate
       conf.corresponding_proxies << proxy
 
       # In the nested section, can't get plugin class through proxies so get plugin class here
-      plugin_class = Fluent::Plugin.lookup_name_from_class(proxy.name.to_s)
+      plugin_class = Fluent::Plugin.lookup_type_from_class(proxy.name.to_s)
       root = Fluent::Config::SectionGenerator.generate(proxy, conf, logger, plugin_class)
       @config_root_section = root
 
       root.instance_eval{ @params.keys }.each do |param_name|
+        next if param_name.to_s.start_with?('@')
         varname = "@#{param_name}".to_sym
-        if (! root[param_name].nil?) || instance_variable_get(varname).nil?
-          instance_variable_set(varname, root[param_name])
-        end
+        instance_variable_set(varname, root[param_name])
       end
 
       self
@@ -82,6 +118,21 @@ module Fluent
       CONFIG_TYPE_REGISTRY.lookup(type)
     end
 
+    {
+      string: Config::STRING_TYPE,
+      enum: Config::ENUM_TYPE,
+      integer: Config::INTEGER_TYPE,
+      float: Config::FLOAT_TYPE,
+      size: Config::SIZE_TYPE,
+      bool: Config::BOOL_TYPE,
+      time: Config::TIME_TYPE,
+      hash: Config::HASH_TYPE,
+      array: Config::ARRAY_TYPE,
+      regexp: Config::REGEXP_TYPE,
+    }.each do |name, type|
+      register_type(name, type)
+    end
+
     module ClassMethods
       def configure_proxy_map
         map = {}
@@ -92,15 +143,21 @@ module Fluent
       def configure_proxy(mod_name)
         map = configure_proxy_map
         unless map[mod_name]
-          proxy = Fluent::Config::ConfigureProxy.new(mod_name, required: true, multi: false)
+          type_lookup = ->(type) { Fluent::Configurable.lookup_type(type) }
+          proxy = Fluent::Config::ConfigureProxy.new(mod_name, root: true, required: true, multi: false, type_lookup: type_lookup)
           map[mod_name] = proxy
         end
         map[mod_name]
       end
 
-      def config_param(name, *args, &block)
-        configure_proxy(self.name).config_param(name, *args, &block)
-        attr_accessor name
+      def configured_in(section_name)
+        configure_proxy(self.name).configured_in(section_name)
+      end
+
+      def config_param(name, type = nil, **kwargs, &block)
+        configure_proxy(self.name).config_param(name, type, **kwargs, &block)
+        # reserved names '@foo' are invalid as attr_accessor name
+        attr_accessor(name) unless kwargs[:skip_accessor] || Fluent::Config::Element::RESERVED_PARAMETERS.include?(name.to_s)
       end
 
       def config_set_default(name, defval)
@@ -111,9 +168,13 @@ module Fluent
         configure_proxy(self.name).config_set_desc(name, desc)
       end
 
-      def config_section(name, *args, &block)
-        configure_proxy(self.name).config_section(name, *args, &block)
-        attr_accessor configure_proxy(self.name).sections[name].param_name
+      def config_section(name, **kwargs, &block)
+        section_already_exists = !!merged_configure_proxy.sections[name]
+        configure_proxy(self.name).config_section(name, **kwargs, &block)
+        variable_name = configure_proxy(self.name).sections[name].variable_name
+        if !section_already_exists && !self.respond_to?(variable_name)
+          attr_accessor variable_name
+        end
       end
 
       def desc(description)
@@ -132,12 +193,9 @@ module Fluent
         configurables.map{ |a| a.configure_proxy(a.name || a.object_id.to_s) }.reduce(:merge)
       end
 
-      def dump(level = 0)
-        configure_proxy_map[self.to_s].dump(level)
+      def dump_config_definition
+        configure_proxy_map[self.to_s].dump_config_definition
       end
     end
   end
-
-  # load default types
-  require 'fluent/config/types'
 end

data/lib/fluent/counter/base_socket.rb

@@ -0,0 +1,46 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'cool.io'
+require 'fluent/msgpack_factory'
+
+module Fluent
+  module Counter
+    class BaseSocket < Coolio::TCPSocket
+      include Fluent::MessagePackFactory::Mixin
+
+      def packed_write(data)
+        write pack(data)
+      end
+
+      def on_read(data)
+        msgpack_unpacker.feed_each(data) do |d|
+          on_message d
+        end
+      end
+
+      def on_message(data)
+        raise NotImplementedError
+      end
+
+      private
+
+      def pack(data)
+        msgpack_packer.pack(data)
+      end
+    end
+  end
+end