fluentd222 1.16.2-x86_64-linux

data/lib/fluent/config/element.rb

@@ -0,0 +1,265 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config/error'
+require 'fluent/config/literal_parser'
+
+module Fluent
+  module Config
+    class Element < Hash
+      def initialize(name, arg, attrs, elements, unused = nil)
+        @name = name
+        @arg = arg
+        @elements = elements
+        super()
+        attrs.each { |k, v|
+          self[k] = v
+        }
+        @unused = unused || attrs.keys
+        @v1_config = false
+        @corresponding_proxies = [] # some plugins use flat parameters, e.g. in_http doesn't provide <format> section for parser.
+        @unused_in = nil # if this element is not used in plugins, correspoing plugin name and parent element name is set, e.g. [source, plugin class].
+
+        # it's global logger, not plugin logger: deprecated message should be global warning, not plugin level.
+        @logger = defined?($log) ? $log : nil
+
+        @target_worker_ids = []
+      end
+
+      attr_accessor :name, :arg, :unused, :v1_config, :corresponding_proxies, :unused_in
+      attr_writer :elements
+      attr_reader :target_worker_ids
+
+      RESERVED_PARAMETERS_COMPAT = {
+        '@type' => 'type',
+        '@id' => 'id',
+        '@log_level' => 'log_level',
+        '@label' => nil,
+      }
+      RESERVED_PARAMETERS = RESERVED_PARAMETERS_COMPAT.keys
+
+      def elements(*names, name: nil, arg: nil)
+        raise ArgumentError, "name and names are exclusive" if name && !names.empty?
+        raise ArgumentError, "arg is available only with name" if arg && !name
+
+        if name
+          @elements.select{|e| e.name == name && (!arg || e.arg == arg) }
+        elsif !names.empty?
+          @elements.select{|e| names.include?(e.name) }
+        else
+          @elements
+        end
+      end
+
+      def add_element(name, arg = '')
+        e = Element.new(name, arg, {}, [])
+        e.v1_config = @v1_config
+        @elements << e
+        e
+      end
+
+      def inspect
+        attrs = super
+        "name:#{@name}, arg:#{@arg}, " + attrs + ", " + @elements.inspect
+      end
+
+      # Used by PP and Pry
+      def pretty_print(q)
+        q.text(inspect)
+      end
+
+      # This method assumes _o_ is an Element object. Should return false for nil or other object
+      def ==(o)
+        self.name == o.name && self.arg == o.arg &&
+          self.keys.size == o.keys.size &&
+          self.keys.reduce(true){|r, k| r && self[k] == o[k] } &&
+          self.elements.size == o.elements.size &&
+          [self.elements, o.elements].transpose.reduce(true){|r, e| r && e[0] == e[1] }
+      end
+
+      def +(o)
+        e = Element.new(@name.dup, @arg.dup, o.merge(self), @elements + o.elements, (@unused + o.unused).uniq)
+        e.v1_config = @v1_config
+        e
+      end
+
+      # no code in fluentd uses this method
+      def each_element(*names, &block)
+        if names.empty?
+          @elements.each(&block)
+        else
+          @elements.each { |e|
+            if names.include?(e.name)
+              block.yield(e)
+            end
+          }
+        end
+      end
+
+      def has_key?(key)
+        @unused_in = [] # some sections, e.g. <store> in copy, is not defined by config_section so clear unused flag for better warning message in check_not_fetched.
+        @unused.delete(key)
+        super
+      end
+
+      def [](key)
+        @unused_in = [] # ditto
+        @unused.delete(key)
+
+        if RESERVED_PARAMETERS.include?(key) && !has_key?(key) && has_key?(RESERVED_PARAMETERS_COMPAT[key])
+          @logger.warn "'#{RESERVED_PARAMETERS_COMPAT[key]}' is deprecated parameter name. use '#{key}' instead." if @logger
+          return self[RESERVED_PARAMETERS_COMPAT[key]]
+        end
+
+        super
+      end
+
+      def check_not_fetched(&block)
+        each_key { |key|
+          if @unused.include?(key)
+            block.call(key, self)
+          end
+        }
+        @elements.each { |e|
+          e.check_not_fetched(&block)
+        }
+      end
+
+      def to_s(nest = 0)
+        indent = "  " * nest
+        nindent = "  " * (nest + 1)
+        out = ""
+        if @arg.nil? || @arg.empty?
+          out << "#{indent}<#{@name}>\n"
+        else
+          out << "#{indent}<#{@name} #{@arg}>\n"
+        end
+        each_pair { |k, v|
+          out << dump_value(k, v, nindent)
+        }
+        @elements.each { |e|
+          out << e.to_s(nest + 1)
+        }
+        out << "#{indent}</#{@name}>\n"
+        out
+      end
+
+      def to_masked_element
+        new_elems = @elements.map { |e| e.to_masked_element }
+        new_elem = Element.new(@name, @arg, {}, new_elems, @unused)
+        new_elem.v1_config = @v1_config
+        new_elem.corresponding_proxies = @corresponding_proxies
+        each_pair { |k, v|
+          new_elem[k] = secret_param?(k) ? 'xxxxxx' : v
+        }
+        new_elem
+      end
+
+      def secret_param?(key)
+        return false if @corresponding_proxies.empty?
+
+        param_key = key.to_sym
+        @corresponding_proxies.each { |proxy|
+          _block, opts = proxy.params[param_key]
+          if opts && opts.has_key?(:secret)
+            return opts[:secret]
+          end
+        }
+
+        false
+      end
+
+      def param_type(key)
+        return nil if @corresponding_proxies.empty?
+
+        param_key = key.to_sym
+        proxy = @corresponding_proxies.detect do |_proxy|
+          _proxy.params.has_key?(param_key)
+        end
+        return nil unless proxy
+        _block, opts = proxy.params[param_key]
+        opts[:type]
+      end
+
+      def default_value(key)
+        return nil if @corresponding_proxies.empty?
+
+        param_key = key.to_sym
+        proxy = @corresponding_proxies.detect do |_proxy|
+          _proxy.params.has_key?(param_key)
+        end
+        return nil unless proxy
+        proxy.defaults[param_key]
+      end
+
+      def dump_value(k, v, nindent)
+        return "#{nindent}#{k} xxxxxx\n" if secret_param?(k)
+        return "#{nindent}#{k} #{v}\n" unless @v1_config
+
+        # for v1 config
+        if v.nil?
+          "#{nindent}#{k} \n"
+        elsif v == :default
+          "#{nindent}#{k} #{default_value(k)}\n"
+        else
+          case param_type(k)
+          when :string
+            "#{nindent}#{k} \"#{self.class.unescape_parameter(v)}\"\n"
+          when :enum, :integer, :float, :size, :bool, :time
+            "#{nindent}#{k} #{v}\n"
+          when :hash, :array
+            "#{nindent}#{k} #{v}\n"
+          else
+            # Unknown type
+            "#{nindent}#{k} #{v}\n"
+          end
+        end
+      end
+
+      def self.unescape_parameter(v)
+        result = ''
+        v.each_char { |c| result << LiteralParser.unescape_char(c) }
+        result
+      end
+
+      def set_target_worker_id(worker_id)
+        @target_worker_ids = [worker_id]
+        @elements.each { |e|
+          e.set_target_worker_id(worker_id)
+        }
+      end
+
+      def set_target_worker_ids(worker_ids)
+        @target_worker_ids = worker_ids.uniq
+        @elements.each { |e|
+          e.set_target_worker_ids(worker_ids.uniq)
+        }
+      end
+
+      def for_every_workers?
+        @target_worker_ids.empty?
+      end
+
+      def for_this_worker?
+        @target_worker_ids.include?(Fluent::Engine.worker_id)
+      end
+
+      def for_another_worker?
+        !@target_worker_ids.empty? && !@target_worker_ids.include?(Fluent::Engine.worker_id)
+      end
+    end
+  end
+end

data/lib/fluent/config/error.rb

@@ -0,0 +1,44 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  class ConfigError < StandardError
+  end
+
+  class ConfigParseError < ConfigError
+  end
+
+  class ObsoletedParameterError < ConfigError
+  end
+
+  class SetNil < Exception
+  end
+
+  class SetDefault < Exception
+  end
+
+  class NotFoundPluginError < ConfigError
+    attr_reader :type, :kind
+
+    def initialize(msg, type: nil, kind: nil)
+      @msg = msg
+      @type = type
+      @kind = kind
+
+      super(msg)
+    end
+  end
+end

data/lib/fluent/config/literal_parser.rb

@@ -0,0 +1,286 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'stringio'
+
+require 'json'
+require 'yajl'
+require 'socket'
+require 'ripper'
+
+require 'fluent/config/basic_parser'
+
+module Fluent
+  module Config
+    class LiteralParser < BasicParser
+      def self.unescape_char(c)
+        case c
+        when '"'
+          '\"'
+        when "'"
+          "\\'"
+        when '\\'
+          '\\\\'
+        when "\r"
+          '\r'
+        when "\n"
+          '\n'
+        when "\t"
+          '\t'
+        when "\f"
+          '\f'
+        when "\b"
+          '\b'
+        else
+          c
+        end
+      end
+
+      def initialize(strscan, eval_context)
+        super(strscan)
+        @eval_context = eval_context
+        unless @eval_context.respond_to?(:use_nil)
+          def @eval_context.use_nil
+            raise SetNil
+          end
+        end
+        unless @eval_context.respond_to?(:use_default)
+          def @eval_context.use_default
+            raise SetDefault
+          end
+        end
+      end
+
+      def parse_literal(string_boundary_charset = LINE_END)
+        spacing_without_comment
+
+        value = if skip(/\[/)
+                  scan_json(true)
+                elsif skip(/\{/)
+                  scan_json(false)
+                else
+                  scan_string(string_boundary_charset)
+                end
+        value
+      end
+
+      def scan_string(string_boundary_charset = LINE_END)
+        if skip(/\"/)
+          return scan_double_quoted_string
+        elsif skip(/\'/)
+          return scan_single_quoted_string
+        else
+          return scan_nonquoted_string(string_boundary_charset)
+        end
+      end
+
+      def scan_double_quoted_string
+        string = []
+        while true
+          if skip(/\"/)
+            if string.include?(nil)
+              return nil
+            elsif string.include?(:default)
+              return :default
+            else
+              return string.join
+            end
+          elsif check(/[^"]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/)
+            if s = check(/[^\\]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/)
+              string << s
+            end
+            skip(/[^"]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/)
+          elsif s = scan(/\\./)
+            string << eval_escape_char(s[1,1])
+          elsif skip(/\#\{/)
+            string << eval_embedded_code(scan_embedded_code)
+            skip(/\}/)
+          elsif s = scan(/./)
+            string << s
+          else
+            parse_error! "unexpected end of file in a double quoted string"
+          end
+        end
+      end
+
+      def scan_single_quoted_string
+        string = []
+        while true
+          if skip(/\'/)
+            return string.join
+          elsif s = scan(/\\'/)
+            string << "'"
+          elsif s = scan(/\\\\/)
+            string << "\\"
+          elsif s = scan(/./)
+            string << s
+          else
+            parse_error! "unexpected end of file in a single quoted string"
+          end
+        end
+      end
+
+      def scan_nonquoted_string(boundary_charset = LINE_END)
+        charset = /(?!#{boundary_charset})./
+
+        string = []
+        while true
+          if s = scan(/\#/)
+            string << '#'
+          elsif s = scan(charset)
+            string << s
+          else
+            break
+          end
+        end
+
+        if string.empty?
+          return nil
+        end
+
+        string.join
+      end
+
+      def scan_embedded_code
+        src = '"#{'+@ss.rest+"\n=begin\n=end\n}"
+
+        seek = -1
+        while (seek = src.index('}', seek + 1))
+          unless Ripper.sexp(src[0..seek] + '"').nil? # eager parsing until valid expression
+            break
+          end
+        end
+
+        unless seek
+          raise Fluent::ConfigParseError, @ss.rest
+        end
+
+        code = src[3, seek-3]
+
+        if @ss.rest.length < code.length
+          @ss.pos += @ss.rest.length
+          parse_error! "expected end of embedded code but $end"
+        end
+
+        @ss.pos += code.length
+
+        '"#{' + code + '}"'
+      end
+
+      def eval_embedded_code(code)
+        if @eval_context.nil?
+          parse_error! "embedded code is not allowed in this file"
+        end
+        # Add hostname and worker_id to code for preventing unused warnings
+        code = <<EOM + code
+hostname = Socket.gethostname
+worker_id = ENV['SERVERENGINE_WORKER_ID'] || ''
+EOM
+        begin
+          @eval_context.instance_eval(code)
+        rescue SetNil
+          nil
+        rescue SetDefault
+          :default
+        end
+      end
+
+      def eval_escape_char(c)
+        case c
+        when '"'
+          '"'
+        when "'"
+          "'"
+        when "r"
+          "\r"
+        when "n"
+          "\n"
+        when "t"
+          "\t"
+        when "f"
+          "\f"
+        when "b"
+          "\b"
+        when "0"
+          "\0"
+        when /[a-zA-Z0-9]/
+          parse_error! "unexpected back-slash escape character '#{c}'"
+        else  # symbols
+          c
+        end
+      end
+
+      def scan_json(is_array)
+        result = nil
+        # Yajl does not raise ParseError for incomplete json string, like '[1', '{"h"', '{"h":' or '{"h1":1'
+        # This is the reason to use JSON module.
+
+        buffer = (is_array ? "[" : "{")
+        line_buffer = ""
+
+        until result
+          char = getch
+
+          break if char.nil?
+
+          if char == "#"
+            # If this is out of json string literals, this object can be parsed correctly
+            # '{"foo":"bar", #' -> '{"foo":"bar"}' (to check)
+            parsed = nil
+            begin
+              parsed = JSON.parse(buffer + line_buffer.rstrip.sub(/,$/, '') + (is_array ? "]" : "}"))
+            rescue JSON::ParserError
+              # This '#' is in json string literals
+            end
+
+            if parsed
+              # ignore chars as comment before newline
+              while (char = getch) != "\n"
+                # ignore comment char
+              end
+              buffer << line_buffer + "\n"
+              line_buffer = ""
+            else
+              # '#' is a char in json string
+              line_buffer << char
+            end
+
+            next # This char '#' MUST NOT terminate json object.
+          end
+
+          if char == "\n"
+            buffer << line_buffer + "\n"
+            line_buffer = ""
+            next
+          end
+
+          line_buffer << char
+          begin
+            result = JSON.parse(buffer + line_buffer)
+          rescue JSON::ParserError
+            # Incomplete json string yet
+          end
+        end
+
+        unless result
+          parse_error! "got incomplete JSON #{is_array ? 'array' : 'hash'} configuration"
+        end
+
+        JSON.dump(result)
+      end
+    end
+  end
+end

data/lib/fluent/config/parser.rb

@@ -0,0 +1,107 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'uri'
+
+require 'fluent/config/error'
+require 'fluent/config/element'
+
+module Fluent
+  module Config
+    class Parser
+      def self.parse(io, fname, basepath = Dir.pwd)
+        attrs, elems = Parser.new(basepath, io.each_line, fname).parse!(true)
+        Element.new('ROOT', '', attrs, elems)
+      end
+
+      def initialize(basepath, iterator, fname, i = 0)
+        @basepath = basepath
+        @iterator = iterator
+        @i = i
+        @fname = fname
+      end
+
+      def parse!(allow_include, elem_name = nil, attrs = {}, elems = [])
+        while line = @iterator.next
+          line.force_encoding('UTF-8')
+          @i += 1
+          line.lstrip!
+          line.gsub!(/\s*(?:\#.*)?$/,'')
+          if line.empty?
+            next
+          elsif m = /^\<include\s*(.*)\s*\/\>$/.match(line)
+            value = m[1].strip
+            process_include(attrs, elems, value, allow_include)
+          elsif m = /^\<([a-zA-Z0-9_]+)\s*(.+?)?\>$/.match(line)
+            e_name = m[1]
+            e_arg = m[2] || ""
+            e_attrs, e_elems = parse!(false, e_name)
+            elems << Element.new(e_name, e_arg, e_attrs, e_elems)
+          elsif line == "</#{elem_name}>"
+            break
+          elsif m = /^([a-zA-Z0-9_]+)\s*(.*)$/.match(line)
+            key = m[1]
+            value = m[2]
+            if allow_include && key == 'include'
+              process_include(attrs, elems, value)
+            else
+              attrs[key] = value
+            end
+            next
+          else
+            raise ConfigParseError, "parse error at #{@fname} line #{@i}"
+          end
+        end
+
+        return attrs, elems
+      rescue StopIteration
+        return attrs, elems
+      end
+
+      def process_include(attrs, elems, uri, allow_include = true)
+        u = URI.parse(uri)
+        if u.scheme == 'file' || u.path == uri  # file path
+          path = u.path
+          if path[0] != ?/
+            pattern = File.expand_path("#{@basepath}/#{path}")
+          else
+            pattern = path
+          end
+
+          Dir.glob(pattern).sort.each { |entry|
+            basepath = File.dirname(entry)
+            fname = File.basename(entry)
+            File.open(entry) { |f|
+              Parser.new(basepath, f.each_line, fname).parse!(allow_include, nil, attrs, elems)
+            }
+          }
+
+        else
+          basepath = '/'
+          fname = path
+          require 'open-uri'
+          URI.open(uri) {|f|
+            Parser.new(basepath, f.each_line, fname).parse!(allow_include, nil, attrs, elems)
+          }
+        end
+
+      rescue SystemCallError => e
+        raise ConfigParseError, "include error at #{@fname} line #{@i}: #{e.to_s}"
+      end
+    end
+  end
+end
+