dtomasgu-fluentd 1.14.7.pre.dev

data/lib/fluent/config/section.rb

@@ -0,0 +1,272 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'json'
+
+require 'fluent/config/error'
+require 'fluent/config/v1_parser'
+
+module Fluent
+  module Config
+    class Section < BasicObject
+      def self.name
+        'Fluent::Config::Section'
+      end
+
+      def initialize(params = {}, config_element = nil)
+        @klass = 'Fluent::Config::Section'
+        @params = params
+        @corresponding_config_element = config_element
+      end
+
+      alias :object_id :__id__
+
+      def corresponding_config_element
+        @corresponding_config_element
+      end
+
+      def class
+        Section
+      end
+
+      def to_s
+        inspect
+      end
+
+      def inspect
+        "<Fluent::Config::Section #{@params.to_json}>"
+      end
+
+      # Used by PP and Pry
+      def pretty_print(q)
+        q.text(inspect)
+      end
+
+      def nil?
+        false
+      end
+
+      def to_h
+        @params
+      end
+
+      def dup
+        Section.new(@params.dup, @corresponding_config_element.dup)
+      end
+
+      def +(other)
+        Section.new(self.to_h.merge(other.to_h))
+      end
+
+      def instance_of?(mod)
+        @klass == mod.name
+      end
+
+      def kind_of?(mod)
+        @klass == mod.name || BasicObject == mod
+      end
+      alias is_a? kind_of?
+
+      def [](key)
+        @params[key.to_sym]
+      end
+
+      def []=(key, value)
+        @params[key.to_sym] = value
+      end
+
+      def respond_to?(symbol, include_all=false)
+        case symbol
+        when :inspect, :nil?, :to_h, :+, :instance_of?, :kind_of?, :[], :respond_to?, :respond_to_missing?
+          true
+        when :!, :!= , :==, :equal?, :instance_eval, :instance_exec
+          true
+        when :method_missing, :singleton_method_added, :singleton_method_removed, :singleton_method_undefined
+          include_all
+        else
+          false
+        end
+      end
+
+      def respond_to_missing?(symbol, include_private)
+        @params.has_key?(symbol)
+      end
+
+      def method_missing(name, *args)
+        if @params.has_key?(name)
+          @params[name]
+        else
+          ::Kernel.raise ::NoMethodError, "undefined method `#{name}' for #{self.inspect}"
+        end
+      end
+    end
+
+    module SectionGenerator
+      def self.generate(proxy, conf, logger, plugin_class, stack = [], strict_config_value = false)
+        return nil if conf.nil?
+
+        section_stack = ""
+        unless stack.empty?
+          section_stack = ", in section " + stack.join(" > ")
+        end
+
+        section_params = {}
+
+        proxy.defaults.each_pair do |name, defval|
+          varname = name.to_sym
+          section_params[varname] = (defval.dup rescue defval)
+        end
+
+        if proxy.argument
+          unless conf.arg.nil? || conf.arg.empty?
+            key, block, opts = proxy.argument
+            opts = opts.merge(strict: true) if strict_config_value
+
+            if conf.arg == :default
+              unless section_params.has_key?(key)
+                logger.error "config error in:\n#{conf}" if logger
+                raise ConfigError, "'#{key}' doesn't have default value"
+              end
+            else
+              begin
+                section_params[key] = self.instance_exec(conf.arg, opts, key, &block)
+              rescue ConfigError => e
+                logger.error "config error in:\n#{conf}" if logger
+                raise e
+              end
+            end
+          end
+          unless section_params.has_key?(proxy.argument.first)
+            logger.error "config error in:\n#{conf}" if logger # logger should exist, but somethimes it's nil (e.g, in tests)
+            raise ConfigError, "'<#{proxy.name} ARG>' section requires argument" + section_stack
+          end
+          # argument should NOT be deprecated... (argument always has a value: '')
+        end
+
+        proxy.params.each_pair do |name, defval|
+          varname = name.to_sym
+          block, opts = defval
+          opts = opts.merge(strict: true) if strict_config_value
+
+          if conf.has_key?(name.to_s) || opts[:alias] && conf.has_key?(opts[:alias].to_s)
+            val = if conf.has_key?(name.to_s)
+                    conf[name.to_s]
+                  else
+                    conf[opts[:alias].to_s]
+                  end
+
+            if val == :default
+              # default value is already set if it exists
+              unless section_params.has_key?(varname)
+                logger.error "config error in:\n#{conf}" if logger
+                raise ConfigError, "'#{varname}' doesn't have default value"
+              end
+            else
+              begin
+                section_params[varname] = self.instance_exec(val, opts, name, &block)
+              rescue ConfigError => e
+                logger.error "config error in:\n#{conf}" if logger
+                raise e
+              end
+            end
+
+            if section_params[varname].nil?
+              unless proxy.defaults.has_key?(varname) && proxy.defaults[varname].nil?
+                logger.error "config error in:\n#{conf}" if logger
+                raise ConfigError, "'#{name}' parameter is required but nil is specified"
+              end
+            end
+
+            # Source of definitions of deprecated/obsoleted:
+            # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Deprecated_and_obsolete_features
+            #
+            # Deprecated: These deprecated features can still be used, but should be used with caution
+            #             because they are expected to be removed entirely sometime in the future.
+            # Obsoleted: These obsolete features have been entirely removed from JavaScript and can no longer be used.
+            if opts[:deprecated]
+              logger.warn "'#{name}' parameter is deprecated: #{opts[:deprecated]}" if logger
+            end
+            if opts[:obsoleted]
+              logger.error "config error in:\n#{conf}" if logger
+              raise ObsoletedParameterError, "'#{name}' parameter is already removed: #{opts[:obsoleted]}" + section_stack
+            end
+          end
+          unless section_params.has_key?(varname)
+            logger.error "config error in:\n#{conf}" if logger
+            raise ConfigError, "'#{name}' parameter is required" + section_stack
+          end
+        end
+
+        check_unused_section(proxy, conf, plugin_class)
+
+        proxy.sections.each do |name, subproxy|
+          varname = subproxy.variable_name
+          elements = (conf.respond_to?(:elements) ? conf.elements : []).select{ |e| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }
+          if elements.empty? && subproxy.init?
+            if subproxy.argument && !subproxy.defaults.has_key?(subproxy.argument.first)
+              raise ArgumentError, "#{name}: init is specified, but default value of argument is missing"
+            end
+            missing_keys = subproxy.params.keys.select{|param_name| !subproxy.defaults.has_key?(param_name)}
+            if !missing_keys.empty?
+              raise ArgumentError, "#{name}: init is specified, but there're parameters without default values:#{missing_keys.join(',')}"
+            end
+            elements << Fluent::Config::Element.new(subproxy.name.to_s, '', {}, [])
+          end
+
+          # set subproxy for secret option
+          elements.each { |element|
+            element.corresponding_proxies << subproxy
+          }
+
+          if subproxy.required? && elements.size < 1
+            logger.error "config error in:\n#{conf}" if logger
+            raise ConfigError, "'<#{subproxy.name}>' sections are required" + section_stack
+          end
+          if subproxy.multi?
+            section_params[varname] = elements.map{ |e| generate(subproxy, e, logger, plugin_class, stack + [subproxy.name], strict_config_value) }
+          else
+            if elements.size > 1
+              logger.error "config error in:\n#{conf}" if logger
+              raise ConfigError, "'<#{subproxy.name}>' section cannot be written twice or more" + section_stack
+            end
+            section_params[varname] = generate(subproxy, elements.first, logger, plugin_class, stack + [subproxy.name], strict_config_value)
+          end
+        end
+
+        Section.new(section_params, conf)
+      end
+
+      def self.check_unused_section(proxy, conf, plugin_class)
+        elems = conf.respond_to?(:elements) ? conf.elements : []
+        elems.each { |e|
+          next if plugin_class.nil? && Fluent::Config::V1Parser::ELEM_SYMBOLS.include?(e.name) # skip pre-defined non-plugin elements because it doesn't have proxy section
+          next if e.unused_in && e.unused_in.empty? # the section is used at least once
+
+          if proxy.sections.any? { |name, subproxy| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }
+            e.unused_in = []
+          else
+            parent_name = if conf.arg.empty?
+                            conf.name
+                          else
+                            "#{conf.name} #{conf.arg}"
+                          end
+            e.unused_in = [parent_name, plugin_class]
+          end
+        }
+      end
+    end
+  end
+end

data/lib/fluent/config/types.rb

@@ -0,0 +1,249 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'json'
+
+require 'fluent/config/error'
+
+module Fluent
+  module Config
+    def self.reformatted_value(type, val, opts = {}, name = nil)
+      REFORMAT_VALUE.call(type, val, opts, name)
+    end
+
+    def self.size_value(str, opts = {}, name = nil)
+      return nil if str.nil?
+
+      case str.to_s
+      when /([0-9]+)k/i
+        $~[1].to_i * 1024
+      when /([0-9]+)m/i
+        $~[1].to_i * (1024 ** 2)
+      when /([0-9]+)g/i
+        $~[1].to_i * (1024 ** 3)
+      when /([0-9]+)t/i
+        $~[1].to_i * (1024 ** 4)
+      else
+        INTEGER_TYPE.call(str, opts, name)
+      end
+    end
+
+    def self.time_value(str, opts = {}, name = nil)
+      return nil if str.nil?
+
+      case str.to_s
+      when /([0-9]+)s/
+        $~[1].to_i
+      when /([0-9]+)m/
+        $~[1].to_i * 60
+      when /([0-9]+)h/
+        $~[1].to_i * 60 * 60
+      when /([0-9]+)d/
+        $~[1].to_i * 24 * 60 * 60
+      else
+        FLOAT_TYPE.call(str, opts, name)
+      end
+    end
+
+    def self.bool_value(str, opts = {}, name = nil)
+      return nil if str.nil?
+
+      case str.to_s
+      when 'true', 'yes'
+        true
+      when 'false', 'no'
+        false
+      when ''
+        true
+      else
+        # Current parser passes comment without actual values, e.g. "param #foo".
+        # parser should pass empty string in this case but changing behaviour may break existing environment so keep parser behaviour. Just ignore comment value in boolean handling for now.
+        if str.respond_to?('start_with?') && str.start_with?('#')
+          true
+        elsif opts[:strict]
+          raise Fluent::ConfigError, "#{name}: invalid bool value: #{str}"
+        else
+          nil
+        end
+      end
+    end
+
+    def self.regexp_value(str, opts = {}, name = nil)
+      return nil unless str
+
+      return Regexp.compile(str) unless str.start_with?("/")
+      right_slash_position = str.rindex("/")
+      if right_slash_position < str.size - 3
+        raise Fluent::ConfigError, "invalid regexp: missing right slash: #{str}"
+      end
+      options = str[(right_slash_position + 1)..-1]
+      option = 0
+      option |= Regexp::IGNORECASE if options.include?("i")
+      option |= Regexp::MULTILINE if options.include?("m")
+      Regexp.compile(str[1...right_slash_position], option)
+    end
+
+    def self.string_value(val, opts = {}, name = nil)
+      return nil if val.nil?
+
+      v = val.to_s
+      v = v.frozen? ? v.dup : v # config_param can't assume incoming string is mutable
+      v.force_encoding(Encoding::UTF_8)
+    end
+
+    STRING_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.string_value(val, opts, name)
+    }
+
+    def self.symbol_value(val, opts = {}, name = nil)
+      return nil if val.nil? || val.empty?
+
+      val.delete_prefix(":").to_sym
+    end
+
+    SYMBOL_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.symbol_value(val, opts, name)
+    }
+
+    def self.enum_value(val, opts = {}, name = nil)
+      return nil if val.nil?
+
+      s = val.to_sym
+      list = opts[:list]
+      raise "Plugin BUG: config type 'enum' requires :list of symbols" unless list.is_a?(Array) && list.all?{|v| v.is_a? Symbol }
+      unless list.include?(s)
+        raise ConfigError, "valid options are #{list.join(',')} but got #{val}"
+      end
+      s
+    end
+
+    ENUM_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.enum_value(val, opts, name)
+    }
+
+    INTEGER_TYPE = Proc.new { |val, opts = {}, name = nil|
+      if val.nil?
+        nil
+      elsif opts[:strict]
+        begin
+          Integer(val)
+        rescue ArgumentError, TypeError => e
+          raise ConfigError, "#{name}: #{e.message}"
+        end
+      else
+        val.to_i
+      end
+    }
+
+    FLOAT_TYPE = Proc.new { |val, opts = {}, name = nil|
+      if val.nil?
+        nil
+      elsif opts[:strict]
+        begin
+          Float(val)
+        rescue ArgumentError, TypeError => e
+          raise ConfigError, "#{name}: #{e.message}"
+        end
+      else
+        val.to_f
+      end
+    }
+
+    SIZE_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.size_value(val, opts, name)
+    }
+
+    BOOL_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.bool_value(val, opts, name)
+    }
+
+    TIME_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.time_value(val, opts, name)
+    }
+
+    REGEXP_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.regexp_value(val, opts, name)
+    }
+
+    REFORMAT_VALUE = ->(type, value, opts = {}, name = nil) {
+      if value.nil?
+        value
+      else
+        case type
+        when :string  then value.to_s.force_encoding(Encoding::UTF_8)
+        when :integer then INTEGER_TYPE.call(value, opts, name)
+        when :float   then FLOAT_TYPE.call(value, opts, name)
+        when :size then Config.size_value(value, opts, name)
+        when :bool then Config.bool_value(value, opts, name)
+        when :time then Config.time_value(value, opts, name)
+        when :regexp then Config.regexp_value(value, opts, name)
+        when :symbol then Config.symbol_value(value, opts, name)
+        else
+          raise "unknown type in REFORMAT: #{type}"
+        end
+      end
+    }
+
+    def self.hash_value(val, opts = {}, name = nil)
+      return nil if val.nil?
+
+      param = if val.is_a?(String)
+                val.start_with?('{') ? JSON.parse(val) : Hash[val.strip.split(/\s*,\s*/).map{|v| v.split(':', 2)}]
+              else
+                val
+              end
+      if param.class != Hash
+        raise ConfigError, "hash required but got #{val.inspect}"
+      end
+      if opts.empty?
+        param
+      else
+        newparam = {}
+        param.each_pair do |key, value|
+          new_key = opts[:symbolize_keys] ? key.to_sym : key
+          newparam[new_key] = opts[:value_type] ? REFORMAT_VALUE.call(opts[:value_type], value, opts, new_key) : value
+        end
+        newparam
+      end
+    end
+
+    HASH_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.hash_value(val, opts, name)
+    }
+
+    def self.array_value(val, opts = {}, name = nil)
+      return nil if val.nil?
+
+      param = if val.is_a?(String)
+                val.start_with?('[') ? JSON.parse(val) : val.strip.split(/\s*,\s*/)
+              else
+                val
+              end
+      if param.class != Array
+        raise ConfigError, "array required but got #{val.inspect}"
+      end
+      if opts[:value_type]
+        param.map{|v| REFORMAT_VALUE.call(opts[:value_type], v, opts, nil) }
+      else
+        param
+      end
+    end
+
+    ARRAY_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.array_value(val, opts, name)
+    }
+  end
+end

data/lib/fluent/config/v1_parser.rb

@@ -0,0 +1,192 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'strscan'
+require 'uri'
+
+require 'fluent/config/error'
+require 'fluent/config/basic_parser'
+require 'fluent/config/literal_parser'
+require 'fluent/config/element'
+
+module Fluent
+  module Config
+    class V1Parser < LiteralParser
+      ELEMENT_NAME = /[a-zA-Z0-9_]+/
+
+      def self.parse(data, fname, basepath = Dir.pwd, eval_context = nil)
+        ss = StringScanner.new(data)
+        ps = V1Parser.new(ss, basepath, fname, eval_context)
+        ps.parse!
+      end
+
+      def initialize(strscan, include_basepath, fname, eval_context)
+        super(strscan, eval_context)
+        @include_basepath = include_basepath
+        @fname = fname
+        @logger = defined?($log) ? $log : nil
+      end
+
+      def parse!
+        attrs, elems = parse_element(true, nil)
+        root = Element.new('ROOT', '', attrs, elems)
+        root.v1_config = true
+
+        spacing
+        unless eof?
+          parse_error! "expected EOF"
+        end
+
+        return root
+      end
+
+      ELEM_SYMBOLS = ['match', 'source', 'filter', 'system']
+      RESERVED_PARAMS = %W(@type @id @label @log_level)
+
+      def parse_element(root_element, elem_name, attrs = {}, elems = [])
+        while true
+          spacing
+          if eof?
+            if root_element
+              break
+            end
+            parse_error! "expected end tag '</#{elem_name}>' but got end of file"
+          end
+
+          if skip(/\<\//)
+            e_name = scan(ELEMENT_NAME)
+            spacing
+            unless skip(/\>/)
+              parse_error! "expected character in tag name"
+            end
+            unless line_end
+              parse_error! "expected end of line after end tag"
+            end
+            if e_name != elem_name
+              parse_error! "unmatched end tag"
+            end
+            break
+
+          elsif skip(/\</)
+            e_name = scan(ELEMENT_NAME)
+            spacing
+            e_arg = scan_string(/(?:#{ZERO_OR_MORE_SPACING}\>)/)
+            spacing
+            unless skip(/\>/)
+              parse_error! "expected '>'"
+            end
+            unless line_end
+              parse_error! "expected end of line after tag"
+            end
+            e_arg ||= ''
+            # call parse_element recursively
+            e_attrs, e_elems = parse_element(false, e_name)
+            new_e = Element.new(e_name, e_arg, e_attrs, e_elems)
+            new_e.v1_config = true
+            elems << new_e
+
+          elsif root_element && skip(/(\@include|include)#{SPACING}/)
+            if !prev_match.start_with?('@')
+              @logger.warn "'include' is deprecated. Use '@include' instead" if @logger
+            end
+            parse_include(attrs, elems)
+
+          else
+            k = scan_string(SPACING)
+            spacing_without_comment
+            if prev_match.include?("\n") || eof? # support 'tag_mapped' like "without value" configuration
+              attrs[k] = ""
+            else
+              if k == '@include'
+                parse_include(attrs, elems)
+              elsif RESERVED_PARAMS.include?(k)
+                v = parse_literal
+                unless line_end
+                  parse_error! "expected end of line"
+                end
+                attrs[k] = v
+              else
+                if k.start_with?('@')
+                  if root_element || ELEM_SYMBOLS.include?(elem_name)
+                    parse_error! "'@' is the system reserved prefix. Don't use '@' prefix parameter in the configuration: #{k}"
+                  else
+                    # TODO: This is for backward compatibility. It will throw an error in the future.
+                    @logger.warn "'@' is the system reserved prefix. It works in the nested configuration for now but it will be rejected: #{k}" if @logger
+                  end
+                end
+
+                v = parse_literal
+                unless line_end
+                  parse_error! "expected end of line"
+                end
+                attrs[k] = v
+              end
+            end
+          end
+        end
+
+        return attrs, elems
+      end
+
+      def parse_include(attrs, elems)
+        uri = scan_string(LINE_END)
+        eval_include(attrs, elems, uri)
+        line_end
+      end
+
+      def eval_include(attrs, elems, uri)
+        # replace space(s)(' ') with '+' to prevent invalid uri due to space(s).
+        # See: https://github.com/fluent/fluentd/pull/2780#issuecomment-576081212
+        u = URI.parse(uri.gsub(/ /, '+'))
+        if u.scheme == 'file' || (!u.scheme.nil? && u.scheme.length == 1) || u.path == uri.gsub(/ /, '+') # file path
+          # When the Windows absolute path then u.scheme.length == 1
+          # e.g. C:
+          path = URI.decode_www_form_component(u.path)
+          if path[0] != ?/
+            pattern = File.expand_path("#{@include_basepath}/#{path}")
+          else
+            pattern = path
+          end
+          Dir.glob(pattern).sort.each { |entry|
+            basepath = File.dirname(entry)
+            fname = File.basename(entry)
+            data = File.read(entry)
+            data.force_encoding('UTF-8')
+            ss = StringScanner.new(data)
+            V1Parser.new(ss, basepath, fname, @eval_context).parse_element(true, nil, attrs, elems)
+          }
+        else
+          require 'open-uri'
+          basepath = '/'
+          fname = path
+          data = URI.open(uri) { |f| f.read }
+          data.force_encoding('UTF-8')
+          ss = StringScanner.new(data)
+          V1Parser.new(ss, basepath, fname, @eval_context).parse_element(true, nil, attrs, elems)
+        end
+      rescue SystemCallError => e
+        cpe = ConfigParseError.new("include error #{uri} - #{e}")
+        cpe.set_backtrace(e.backtrace)
+        raise cpe
+      end
+
+      # override
+      def error_sample
+        "#{@fname} #{super}"
+      end
+    end
+  end
+end