fluentd 1.20.0-aarch64-mingw-ucrt

data/lib/fluent/config/parser.rb

@@ -0,0 +1,119 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'uri'
+
+require 'fluent/config/error'
+require 'fluent/config/element'
+
+module Fluent
+  module Config
+    class Parser
+      def self.parse(io, fname, basepath = Dir.pwd)
+        attrs, elems = Parser.new(basepath, io.each_line, fname).parse!(true)
+        Element.new('ROOT', '', attrs, elems)
+      end
+
+      def initialize(basepath, iterator, fname, i = 0)
+        @basepath = basepath
+        @iterator = iterator
+        @i = i
+        @fname = fname
+      end
+
+      def parse!(allow_include, elem_name = nil, attrs = {}, elems = [])
+        while line = @iterator.next
+          line.force_encoding('UTF-8')
+          @i += 1
+          line.lstrip!
+          line.gsub!(/\s*(?:\#.*)?$/,'')
+          if line.empty?
+            next
+          elsif m = /^\<include\s*(.*)\s*\/\>$/.match(line)
+            value = m[1].strip
+            process_include(attrs, elems, value, allow_include)
+          elsif m = /^\<([a-zA-Z0-9_]+)\s*(.+?)?\>$/.match(line)
+            e_name = m[1]
+            e_arg = m[2] || ""
+            e_attrs, e_elems = parse!(false, e_name)
+            elems << Element.new(e_name, e_arg, e_attrs, e_elems)
+          elsif line == "</#{elem_name}>"
+            break
+          elsif m = /^([a-zA-Z0-9_]+)\s*(.*)$/.match(line)
+            key = m[1]
+            value = m[2]
+            if allow_include && key == 'include'
+              process_include(attrs, elems, value)
+            else
+              attrs[key] = value
+            end
+            next
+          else
+            raise ConfigParseError, "parse error at #{@fname} line #{@i}"
+          end
+        end
+
+        return attrs, elems
+      rescue StopIteration
+        return attrs, elems
+      end
+
+      def process_include(attrs, elems, uri, allow_include = true)
+        u = URI.parse(uri)
+        if u.scheme == 'file' || u.path == uri  # file path
+          path = u.path
+          if path[0] != ?/
+            pattern = File.expand_path("#{@basepath}/#{path}")
+          else
+            pattern = path
+          end
+
+          Dir.glob(pattern).sort.each { |entry|
+            basepath = File.dirname(entry)
+            fname = File.basename(entry)
+            File.open(entry) { |f|
+              Parser.new(basepath, f.each_line, fname).parse!(allow_include, nil, attrs, elems)
+            }
+          }
+
+        else
+          basepath = '/'
+          fname = path
+          parser_proc = ->(f) {
+            Parser.new(basepath, f.each_line, fname).parse!(allow_include, nil, attrs, elems)
+          }
+
+          case u.scheme
+          when 'http', 'https', 'ftp'
+            # URI#open can be able to handle URIs for http, https and ftp.
+            require 'open-uri'
+            u.open(&parser_proc)
+          else
+            # TODO: This case should be handled in the previous if condition. Glob is not applied to some Windows path formats.
+            # 'c:/path/to/file' will be passed as URI, 'uri' and 'u.path' will be:
+            #   - uri is 'c:/path/to/file'
+            #   - u.path is '/path/to/file' and u.scheme is 'c'
+            # Therefore, the condition of the if statement above is not met and it is handled here.
+            File.open(uri, &parser_proc)
+          end
+        end
+
+      rescue SystemCallError => e
+        raise ConfigParseError, "include error at #{@fname} line #{@i}: #{e.to_s}"
+      end
+    end
+  end
+end

data/lib/fluent/config/section.rb

@@ -0,0 +1,272 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'json'
+
+require 'fluent/config/error'
+require 'fluent/config/v1_parser'
+
+module Fluent
+  module Config
+    class Section < BasicObject
+      def self.name
+        'Fluent::Config::Section'
+      end
+
+      def initialize(params = {}, config_element = nil)
+        @klass = 'Fluent::Config::Section'
+        @params = params
+        @corresponding_config_element = config_element
+      end
+
+      alias :object_id :__id__
+
+      def corresponding_config_element
+        @corresponding_config_element
+      end
+
+      def class
+        Section
+      end
+
+      def to_s
+        inspect
+      end
+
+      def inspect
+        "<Fluent::Config::Section #{@params.to_json}>"
+      end
+
+      # Used by PP and Pry
+      def pretty_print(q)
+        q.text(inspect)
+      end
+
+      def nil?
+        false
+      end
+
+      def to_h
+        @params
+      end
+
+      def dup
+        Section.new(@params.dup, @corresponding_config_element.dup)
+      end
+
+      def +(other)
+        Section.new(self.to_h.merge(other.to_h))
+      end
+
+      def instance_of?(mod)
+        @klass == mod.name
+      end
+
+      def kind_of?(mod)
+        @klass == mod.name || BasicObject == mod
+      end
+      alias is_a? kind_of?
+
+      def [](key)
+        @params[key.to_sym]
+      end
+
+      def []=(key, value)
+        @params[key.to_sym] = value
+      end
+
+      def respond_to?(symbol, include_all=false)
+        case symbol
+        when :inspect, :nil?, :to_h, :+, :instance_of?, :kind_of?, :[], :respond_to?, :respond_to_missing?
+          true
+        when :!, :!= , :==, :equal?, :instance_eval, :instance_exec
+          true
+        when :method_missing, :singleton_method_added, :singleton_method_removed, :singleton_method_undefined
+          include_all
+        else
+          false
+        end
+      end
+
+      def respond_to_missing?(symbol, include_private)
+        @params.has_key?(symbol)
+      end
+
+      def method_missing(name, *args)
+        if @params.has_key?(name)
+          @params[name]
+        else
+          ::Kernel.raise ::NoMethodError, "undefined method `#{name}' for #{self.inspect}"
+        end
+      end
+    end
+
+    module SectionGenerator
+      def self.generate(proxy, conf, logger, plugin_class, stack = [], strict_config_value = false)
+        return nil if conf.nil?
+
+        section_stack = ""
+        unless stack.empty?
+          section_stack = ", in section " + stack.join(" > ")
+        end
+
+        section_params = {}
+
+        proxy.defaults.each_pair do |name, defval|
+          varname = name.to_sym
+          section_params[varname] = (defval.dup rescue defval)
+        end
+
+        if proxy.argument
+          unless conf.arg.nil? || conf.arg.empty?
+            key, block, opts = proxy.argument
+            opts = opts.merge(strict: true) if strict_config_value
+
+            if conf.arg == :default
+              unless section_params.has_key?(key)
+                logger.error "config error in:\n#{conf}" if logger
+                raise ConfigError, "'#{key}' doesn't have default value"
+              end
+            else
+              begin
+                section_params[key] = self.instance_exec(conf.arg, opts, key, &block)
+              rescue ConfigError => e
+                logger.error "config error in:\n#{conf}" if logger
+                raise e
+              end
+            end
+          end
+          unless section_params.has_key?(proxy.argument.first)
+            logger.error "config error in:\n#{conf}" if logger # logger should exist, but sometimes it's nil (e.g, in tests)
+            raise ConfigError, "'<#{proxy.name} ARG>' section requires argument" + section_stack
+          end
+          # argument should NOT be deprecated... (argument always has a value: '')
+        end
+
+        proxy.params.each_pair do |name, defval|
+          varname = name.to_sym
+          block, opts = defval
+          opts = opts.merge(strict: true) if strict_config_value
+
+          if conf.has_key?(name.to_s) || opts[:alias] && conf.has_key?(opts[:alias].to_s)
+            val = if conf.has_key?(name.to_s)
+                    conf[name.to_s]
+                  else
+                    conf[opts[:alias].to_s]
+                  end
+
+            if val == :default
+              # default value is already set if it exists
+              unless section_params.has_key?(varname)
+                logger.error "config error in:\n#{conf}" if logger
+                raise ConfigError, "'#{varname}' doesn't have default value"
+              end
+            else
+              begin
+                section_params[varname] = self.instance_exec(val, opts, name, &block)
+              rescue ConfigError => e
+                logger.error "config error in:\n#{conf}" if logger
+                raise e
+              end
+            end
+
+            if section_params[varname].nil?
+              unless proxy.defaults.has_key?(varname) && proxy.defaults[varname].nil?
+                logger.error "config error in:\n#{conf}" if logger
+                raise ConfigError, "'#{name}' parameter is required but nil is specified"
+              end
+            end
+
+            # Source of definitions of deprecated/obsoleted:
+            # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Deprecated_and_obsolete_features
+            #
+            # Deprecated: These deprecated features can still be used, but should be used with caution
+            #             because they are expected to be removed entirely sometime in the future.
+            # Obsoleted: These obsolete features have been entirely removed from JavaScript and can no longer be used.
+            if opts[:deprecated]
+              logger.warn "'#{name}' parameter is deprecated: #{opts[:deprecated]}" if logger
+            end
+            if opts[:obsoleted]
+              logger.error "config error in:\n#{conf}" if logger
+              raise ObsoletedParameterError, "'#{name}' parameter is already removed: #{opts[:obsoleted]}" + section_stack
+            end
+          end
+          unless section_params.has_key?(varname)
+            logger.error "config error in:\n#{conf}" if logger
+            raise ConfigError, "'#{name}' parameter is required" + section_stack
+          end
+        end
+
+        check_unused_section(proxy, conf, plugin_class)
+
+        proxy.sections.each do |name, subproxy|
+          varname = subproxy.variable_name
+          elements = (conf.respond_to?(:elements) ? conf.elements : []).select{ |e| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }
+          if elements.empty? && subproxy.init?
+            if subproxy.argument && !subproxy.defaults.has_key?(subproxy.argument.first)
+              raise ArgumentError, "#{name}: init is specified, but default value of argument is missing"
+            end
+            missing_keys = subproxy.params.keys.select{|param_name| !subproxy.defaults.has_key?(param_name)}
+            if !missing_keys.empty?
+              raise ArgumentError, "#{name}: init is specified, but there're parameters without default values:#{missing_keys.join(',')}"
+            end
+            elements << Fluent::Config::Element.new(subproxy.name.to_s, '', {}, [])
+          end
+
+          # set subproxy for secret option
+          elements.each { |element|
+            element.corresponding_proxies << subproxy
+          }
+
+          if subproxy.required? && elements.size < 1
+            logger.error "config error in:\n#{conf}" if logger
+            raise ConfigError, "'<#{subproxy.name}>' sections are required" + section_stack
+          end
+          if subproxy.multi?
+            section_params[varname] = elements.map{ |e| generate(subproxy, e, logger, plugin_class, stack + [subproxy.name], strict_config_value) }
+          else
+            if elements.size > 1
+              logger.error "config error in:\n#{conf}" if logger
+              raise ConfigError, "'<#{subproxy.name}>' section cannot be written twice or more" + section_stack
+            end
+            section_params[varname] = generate(subproxy, elements.first, logger, plugin_class, stack + [subproxy.name], strict_config_value)
+          end
+        end
+
+        Section.new(section_params, conf)
+      end
+
+      def self.check_unused_section(proxy, conf, plugin_class)
+        elems = conf.respond_to?(:elements) ? conf.elements : []
+        elems.each { |e|
+          next if plugin_class.nil? && Fluent::Config::V1Parser::ELEM_SYMBOLS.include?(e.name) # skip pre-defined non-plugin elements because it doesn't have proxy section
+          next if e.unused_in&.empty? # the section is used at least once
+
+          if proxy.sections.any? { |name, subproxy| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }
+            e.unused_in = []
+          else
+            parent_name = if conf.arg.empty?
+                            conf.name
+                          else
+                            "#{conf.name} #{conf.arg}"
+                          end
+            e.unused_in = [parent_name, plugin_class]
+          end
+        }
+      end
+    end
+  end
+end

data/lib/fluent/config/types.rb

@@ -0,0 +1,257 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'json'
+
+require 'fluent/config/error'
+
+module Fluent
+  module Config
+    def self.reformatted_value(type, val, opts = {}, name = nil)
+      REFORMAT_VALUE.call(type, val, opts, name)
+    end
+
+    def self.size_value(str, opts = {}, name = nil)
+      return nil if str.nil?
+
+      case str.to_s
+      when /([0-9]+)k/i
+        $~[1].to_i * 1024
+      when /([0-9]+)m/i
+        $~[1].to_i * (1024 ** 2)
+      when /([0-9]+)g/i
+        $~[1].to_i * (1024 ** 3)
+      when /([0-9]+)t/i
+        $~[1].to_i * (1024 ** 4)
+      else
+        INTEGER_TYPE.call(str, opts, name)
+      end
+    end
+
+    def self.time_value(str, opts = {}, name = nil)
+      return nil if str.nil?
+
+      case str.to_s
+      when /([0-9]+)s/
+        $~[1].to_i
+      when /([0-9]+)m/
+        $~[1].to_i * 60
+      when /([0-9]+)h/
+        $~[1].to_i * 60 * 60
+      when /([0-9]+)d/
+        $~[1].to_i * 24 * 60 * 60
+      else
+        FLOAT_TYPE.call(str, opts, name)
+      end
+    end
+
+    def self.bool_value(str, opts = {}, name = nil)
+      return nil if str.nil?
+
+      case str.to_s
+      when 'true', 'yes'
+        true
+      when 'false', 'no'
+        false
+      when ''
+        true
+      else
+        # Current parser passes comment without actual values, e.g. "param #foo".
+        # parser should pass empty string in this case but changing behaviour may break existing environment so keep parser behaviour. Just ignore comment value in boolean handling for now.
+        if str.respond_to?(:start_with?) && str.start_with?('#')
+          true
+        elsif opts[:strict]
+          raise Fluent::ConfigError, "#{name}: invalid bool value: #{str}"
+        else
+          nil
+        end
+      end
+    end
+
+    def self.regexp_value(str, opts = {}, name = nil)
+      return nil unless str
+
+      return Regexp.compile(str) unless str.start_with?("/")
+      right_slash_position = str.rindex("/")
+      if right_slash_position < str.size - 3
+        raise Fluent::ConfigError, "invalid regexp: missing right slash: #{str}"
+      end
+      options = str[(right_slash_position + 1)..-1]
+      option = 0
+      option |= Regexp::IGNORECASE if options.include?("i")
+      option |= Regexp::MULTILINE if options.include?("m")
+      Regexp.compile(str[1...right_slash_position], option)
+    end
+
+    def self.string_value(val, opts = {}, name = nil)
+      return nil if val.nil?
+
+      v = val.to_s
+      v = v.frozen? ? v.dup : v # config_param can't assume incoming string is mutable
+      v.force_encoding(Encoding::UTF_8)
+    end
+
+    STRING_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.string_value(val, opts, name)
+    }
+
+    def self.symbol_value(val, opts = {}, name = nil)
+      return nil if val.nil? || val.empty?
+
+      val.delete_prefix(":").to_sym
+    end
+
+    SYMBOL_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.symbol_value(val, opts, name)
+    }
+
+    def self.enum_value(val, opts = {}, name = nil)
+      return nil if val.nil?
+
+      s = val.to_sym
+      list = opts[:list]
+      raise "Plugin BUG: config type 'enum' requires :list of symbols" unless list.is_a?(Array) && list.all?(Symbol)
+      unless list.include?(s)
+        raise ConfigError, "valid options are #{list.join(',')} but got #{val}"
+      end
+      s
+    end
+
+    ENUM_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.enum_value(val, opts, name)
+    }
+
+    INTEGER_TYPE = Proc.new { |val, opts = {}, name = nil|
+      if val.nil?
+        nil
+      elsif opts[:strict]
+        begin
+          Integer(val)
+        rescue ArgumentError, TypeError => e
+          raise ConfigError, "#{name}: #{e.message}"
+        end
+      else
+        val.to_i
+      end
+    }
+
+    FLOAT_TYPE = Proc.new { |val, opts = {}, name = nil|
+      if val.nil?
+        nil
+      elsif opts[:strict]
+        begin
+          Float(val)
+        rescue ArgumentError, TypeError => e
+          raise ConfigError, "#{name}: #{e.message}"
+        end
+      else
+        val.to_f
+      end
+    }
+
+    SIZE_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.size_value(val, opts, name)
+    }
+
+    BOOL_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.bool_value(val, opts, name)
+    }
+
+    TIME_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.time_value(val, opts, name)
+    }
+
+    REGEXP_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.regexp_value(val, opts, name)
+    }
+
+    REFORMAT_VALUE = ->(type, value, opts = {}, name = nil) {
+      if value.nil?
+        value
+      else
+        case type
+        when :string  then value.to_s.force_encoding(Encoding::UTF_8)
+        when :integer then INTEGER_TYPE.call(value, opts, name)
+        when :float   then FLOAT_TYPE.call(value, opts, name)
+        when :size then Config.size_value(value, opts, name)
+        when :bool then Config.bool_value(value, opts, name)
+        when :time then Config.time_value(value, opts, name)
+        when :regexp then Config.regexp_value(value, opts, name)
+        when :symbol then Config.symbol_value(value, opts, name)
+        else
+          raise "unknown type in REFORMAT: #{type}"
+        end
+      end
+    }
+
+    def self.hash_value(val, opts = {}, name = nil)
+      return nil if val.nil?
+
+      param = if val.is_a?(String)
+                val.start_with?('{') ? JSON.parse(val, **Fluent::DEFAULT_JSON_PARSE_OPTIONS) : Hash[val.strip.split(/\s*,\s*/).map{|v| v.split(':', 2)}]
+              else
+                val
+              end
+      if param.class != Hash
+        raise ConfigError, "hash required but got #{val.inspect}"
+      end
+      if opts.empty?
+        param
+      else
+        newparam = {}
+        param.each_pair do |key, value|
+          new_key = opts[:symbolize_keys] ? key.to_sym : key
+          newparam[new_key] = opts[:value_type] ? REFORMAT_VALUE.call(opts[:value_type], value, opts, new_key) : value
+        end
+        newparam
+      end
+    end
+
+    HASH_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.hash_value(val, opts, name)
+    }
+
+    def self.array_value(val, opts = {}, name = nil)
+      return nil if val.nil?
+
+      param = if val.is_a?(String)
+                val.start_with?('[') ? JSON.parse(val, **Fluent::DEFAULT_JSON_PARSE_OPTIONS) : val.strip.split(/\s*,\s*/)
+              elsif val.is_a?(Array)
+                val
+              elsif val.is_a?(Numeric) || val == true || val == false
+                # Wrap only the bare scalars that Psych/YAML actually produces
+                # here (nil is handled by the early return above). Any other
+                # type (Hash, Symbol, Time, arbitrary objects) falls through and
+                # still raises "array required" below.
+                [val]
+              else
+                val
+              end
+      if param.class != Array
+        raise ConfigError, "array required but got #{val.inspect}"
+      end
+      if opts[:value_type]
+        param.map{|v| REFORMAT_VALUE.call(opts[:value_type], v, opts, nil) }
+      else
+        param
+      end
+    end
+
+    ARRAY_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.array_value(val, opts, name)
+    }
+  end
+end