fluentd 1.20.0-aarch64-mingw-ucrt

data/lib/fluent/config/v1_parser.rb

@@ -0,0 +1,200 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'strscan'
+require 'uri'
+
+require 'fluent/config/error'
+require 'fluent/config/basic_parser'
+require 'fluent/config/literal_parser'
+require 'fluent/config/element'
+
+module Fluent
+  module Config
+    class V1Parser < LiteralParser
+      ELEMENT_NAME = /[a-zA-Z0-9_]+/
+
+      def self.parse(data, fname, basepath = Dir.pwd, eval_context = nil, on_file_parsed: nil)
+        ss = StringScanner.new(data)
+        ps = V1Parser.new(ss, basepath, fname, eval_context, on_file_parsed: on_file_parsed)
+        ps.parse!
+      end
+
+      def initialize(strscan, include_basepath, fname, eval_context, on_file_parsed: nil)
+        super(strscan, eval_context)
+        @include_basepath = include_basepath
+        @fname = fname
+        @logger = defined?($log) ? $log : nil
+        @on_file_parsed = on_file_parsed
+      end
+
+      def parse!
+        attrs, elems = parse_element(true, nil)
+        root = Element.new('ROOT', '', attrs, elems)
+        root.v1_config = true
+
+        spacing
+        unless eof?
+          parse_error! "expected EOF"
+        end
+
+        return root
+      end
+
+      ELEM_SYMBOLS = ['match', 'source', 'filter', 'system']
+      RESERVED_PARAMS = %W(@type @id @label @log_level)
+
+      def parse_element(root_element, elem_name, attrs = {}, elems = [])
+        while true
+          spacing
+          if eof?
+            if root_element
+              break
+            end
+            parse_error! "expected end tag '</#{elem_name}>' but got end of file"
+          end
+
+          if skip(/\<\//)
+            e_name = scan(ELEMENT_NAME)
+            spacing
+            unless skip(/\>/)
+              parse_error! "expected character in tag name"
+            end
+            unless line_end
+              parse_error! "expected end of line after end tag"
+            end
+            if e_name != elem_name
+              parse_error! "unmatched end tag"
+            end
+            break
+
+          elsif skip(/\</)
+            e_name = scan(ELEMENT_NAME)
+            spacing
+            e_arg = scan_string(/(?:#{ZERO_OR_MORE_SPACING}\>)/o)
+            spacing
+            unless skip(/\>/)
+              parse_error! "expected '>'"
+            end
+            unless line_end
+              parse_error! "expected end of line after tag"
+            end
+            e_arg ||= ''
+            # call parse_element recursively
+            e_attrs, e_elems = parse_element(false, e_name)
+            new_e = Element.new(e_name, e_arg, e_attrs, e_elems)
+            new_e.v1_config = true
+            elems << new_e
+
+          elsif root_element && skip(/(\@include|include)#{SPACING}/o)
+            if !prev_match.start_with?('@')
+              @logger.warn "'include' is deprecated. Use '@include' instead" if @logger
+            end
+            parse_include(attrs, elems)
+
+          else
+            k = scan_string(SPACING)
+            spacing_without_comment
+            if prev_match.include?("\n") || eof? # support 'tag_mapped' like "without value" configuration
+              attrs[k] = ""
+            else
+              if k == '@include'
+                parse_include(attrs, elems)
+              elsif RESERVED_PARAMS.include?(k)
+                v = parse_literal
+                unless line_end
+                  parse_error! "expected end of line"
+                end
+                attrs[k] = v
+              else
+                if k.start_with?('@')
+                  if root_element || ELEM_SYMBOLS.include?(elem_name)
+                    parse_error! "'@' is the system reserved prefix. Don't use '@' prefix parameter in the configuration: #{k}"
+                  else
+                    # TODO: This is for backward compatibility. It will throw an error in the future.
+                    @logger.warn "'@' is the system reserved prefix. It works in the nested configuration for now but it will be rejected: #{k}" if @logger
+                  end
+                end
+
+                v = parse_literal
+                unless line_end
+                  parse_error! "expected end of line"
+                end
+                attrs[k] = v
+              end
+            end
+          end
+        end
+
+        @on_file_parsed&.call(File.expand_path(File.join(@include_basepath, @fname))) if root_element
+
+        return attrs, elems
+      end
+
+      def parse_include(attrs, elems)
+        uri = scan_string(LINE_END)
+        eval_include(attrs, elems, uri)
+        line_end
+      end
+
+      def eval_include(attrs, elems, uri)
+        # replace space(s)(' ') with '+' to prevent invalid uri due to space(s).
+        # See: https://github.com/fluent/fluentd/pull/2780#issuecomment-576081212
+        u = URI.parse(uri.tr(' ', '+'))
+        if u.scheme == 'file' || (!u.scheme.nil? && u.scheme.length == 1) || u.path == uri.tr(' ', '+') # file path
+          # When the Windows absolute path then u.scheme.length == 1
+          # e.g. C:
+          path = URI.decode_www_form_component(u.path)
+          if path[0] != ?/
+            pattern = File.expand_path("#{@include_basepath}/#{path}")
+          else
+            pattern = path
+          end
+          Dir.glob(pattern).sort.each { |entry|
+            basepath = File.dirname(entry)
+            fname = File.basename(entry)
+            suspicious_backup_extensions = %w(bak old backup orig prev conf tmp temp debug wip)
+            if path.end_with?('*.conf') and
+              suspicious_backup_extensions.any? { |ext| fname.end_with?(".#{ext}.conf", "_#{ext}.conf") }
+              @logger.warn "There is a possibility that '@include #{uri}' includes duplicated backed-up config file such as <#{fname}>" if @logger
+            end
+            data = File.read(entry)
+            data.force_encoding('UTF-8')
+            ss = StringScanner.new(data)
+            V1Parser.new(ss, basepath, fname, @eval_context, on_file_parsed: @on_file_parsed).parse_element(true, nil, attrs, elems)
+          }
+        else
+          require 'open-uri'
+          basepath = '/'
+          fname = path
+          data = u.open { |f| f.read }
+          data.force_encoding('UTF-8')
+          ss = StringScanner.new(data)
+          V1Parser.new(ss, basepath, fname, @eval_context, on_file_parsed: @on_file_parsed).parse_element(true, nil, attrs, elems)
+        end
+      rescue SystemCallError => e
+        cpe = ConfigParseError.new("include error #{uri} - #{e}")
+        cpe.set_backtrace(e.backtrace)
+        raise cpe
+      end
+
+      # override
+      def error_sample
+        "#{@fname} #{super}"
+      end
+    end
+  end
+end

data/lib/fluent/config/yaml_parser/fluent_value.rb

@@ -0,0 +1,47 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Config
+    module YamlParser
+      module FluentValue
+        JsonValue = Struct.new(:val) do
+          def to_s
+            val.to_json
+          end
+
+          def to_element
+            to_s
+          end
+        end
+
+        StringValue = Struct.new(:val, :context) do
+          def to_s
+            context.instance_eval("\"#{val}\"")
+          rescue Fluent::SetNil => _
+            ''
+          rescue Fluent::SetDefault => _
+            ':default'
+          end
+
+          def to_element
+            to_s
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/config/yaml_parser/loader.rb

@@ -0,0 +1,124 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'psych'
+require 'json'
+require 'fluent/config/error'
+require 'fluent/config/yaml_parser/fluent_value'
+
+# Based on https://github.com/eagletmt/hako/blob/34cdde06fe8f3aeafd794be830180c3cedfbb4dc/lib/hako/yaml_loader.rb
+
+module Fluent
+  module Config
+    module YamlParser
+      class Loader
+        INCLUDE_TAG = 'tag:include'.freeze
+        FLUENT_JSON_TAG = 'tag:fluent/json'.freeze
+        FLUENT_STR_TAG = 'tag:fluent/s'.freeze
+        SHOVEL = '<<'.freeze
+
+        def initialize(context = Kernel.binding, on_file_parsed: nil)
+          @context = context
+          @current_path = nil
+          @on_file_parsed = on_file_parsed
+        end
+
+        # @param [String] path
+        # @return [Hash]
+        def load(path)
+          class_loader = Psych::ClassLoader.new
+          scanner = Psych::ScalarScanner.new(class_loader)
+
+          visitor = Visitor.new(scanner, class_loader)
+
+          visitor._register_domain(INCLUDE_TAG) do |_, val|
+            eval_include(Pathname.new(val), path.parent)
+          end
+
+          visitor._register_domain(FLUENT_JSON_TAG) do |_, val|
+            Fluent::Config::YamlParser::FluentValue::JsonValue.new(val)
+          end
+
+          visitor._register_domain(FLUENT_STR_TAG) do |_, val|
+            Fluent::Config::YamlParser::FluentValue::StringValue.new(val, @context)
+          end
+
+          config = path.open do |f|
+            visitor.accept(Psych.parse(f))
+          end
+
+          @on_file_parsed&.call(File.expand_path(path.to_s))
+
+          config
+        end
+
+        def eval_include(path, parent)
+          if path.relative?
+            pattern = parent.join(path)
+          else
+            pattern = path
+          end
+          # A single file keeps its own shape; only glob matches are aggregated.
+          return load(pattern) if File.file?(pattern)
+
+          result = []
+          Dir.glob(pattern).sort.each do |path|
+            loaded = load(Pathname.new(path))
+            if loaded.is_a?(Hash)
+              result.push(loaded)
+            else
+              result.concat(loaded)
+            end
+          end
+          result
+        rescue SystemCallError => e
+          parse_error = ConfigParseError.new("include error #{path} - #{e}")
+          parse_error.set_backtrace(e.backtrace)
+          raise parse_error
+        end
+
+        class Visitor < Psych::Visitors::ToRuby
+          def initialize(scanner, class_loader)
+            super(scanner, class_loader)
+          end
+
+          def _register_domain(name, &block)
+            @domain_types.merge!({ name => [name, block] })
+          end
+
+          def revive_hash(hash, o)
+            super(hash, o).tap do |r|
+              if r[SHOVEL].is_a?(Array) && r[SHOVEL].all?(Hash)
+                r[SHOVEL] = r[SHOVEL].reverse_each.reduce({}) { |a, e| a.merge!(e) }
+              end
+              if r[SHOVEL].is_a?(Hash)
+                h2 = {}
+                r.each do |k, v|
+                  if k == SHOVEL
+                    h2.merge!(v)
+                  else
+                    h2[k] = v
+                  end
+                end
+                r.replace(h2)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/config/yaml_parser/parser.rb

@@ -0,0 +1,179 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config/yaml_parser/section_builder'
+
+module Fluent
+  module Config
+    module YamlParser
+      class Parser
+        def initialize(config, indent: 2)
+          @base_indent = indent
+          @config = config
+        end
+
+        def build
+          s = @config['system'] && system_config_build(@config['system'])
+          c = @config['config'] && config_build(@config['config'], root: true)
+          RootBuilder.new(s, c)
+        end
+
+        private
+
+        def system_config_build(config)
+          section_build('system', config)
+        end
+
+        def config_build(config, indent: 0, root: false)
+          sb = SectionBodyBuilder.new(indent, root: root)
+          config.each do |c|
+            if (lc = c.delete('label'))
+              sb.add_section(label_build(lc, indent: indent))
+            end
+
+            if (sc = c.delete('source'))
+              sb.add_section(source_build(sc, indent: indent))
+            end
+
+            if (fc = c.delete('filter'))
+              sb.add_section(filter_build(fc, indent: indent))
+            end
+
+            if (mc = c.delete('match'))
+              sb.add_section(match_build(mc, indent: indent))
+            end
+
+            if (wc = c.delete('worker'))
+              sb.add_section(worker_build(wc, indent: indent))
+            end
+
+            included_sections_build(c, sb, indent: indent)
+          end
+
+          sb
+        end
+
+        def label_build(config, indent: 0)
+          config = config.dup
+          name = config.delete('$name')
+          c = config.delete('config')
+          SectionBuilder.new('label', config_build(c, indent: indent + @base_indent), indent, name)
+        end
+
+        def worker_build(config, indent: 0)
+          config = config.dup
+          num = config.delete('$arg')
+          c = config.delete('config')
+          SectionBuilder.new('worker', config_build(c, indent: indent + @base_indent), indent, num)
+        end
+
+        def source_build(config, indent: 0)
+          section_build('source', config, indent: indent)
+        end
+
+        def filter_build(config, indent: 0)
+          config = config.dup
+          tag = config.delete('$tag')
+          if tag.is_a?(Array)
+            section_build('filter', config, indent: indent, arg: tag&.join(','))
+          else
+            section_build('filter', config, indent: indent, arg: tag)
+          end
+        end
+
+        def match_build(config, indent: 0)
+          config = config.dup
+          tag = config.delete('$tag')
+          if tag.is_a?(Array)
+            section_build('match', config, indent: indent, arg: tag&.join(','))
+          else
+            section_build('match', config, indent: indent, arg: tag)
+          end
+        end
+
+        def included_sections_build(config, section_builder, indent: 0)
+          config.each_entry do |e|
+            e.keys.each do |k|
+              cc = e.delete(k)
+              case k
+              when 'label'
+                section_builder.add_section(label_build(cc, indent: indent))
+              when 'worker'
+                section_builder.add_section(worker_build(cc, indent: indent))
+              when 'source'
+                section_builder.add_section(source_build(cc, indent: indent))
+              when 'filter'
+                section_builder.add_section(filter_build(cc, indent: indent))
+              when 'match'
+                section_builder.add_section(match_build(cc, indent: indent))
+              end
+            end
+          end
+        end
+
+        def section_build(name, config, indent: 0, arg: nil)
+          sb = SectionBodyBuilder.new(indent + @base_indent)
+
+          if (v = config.delete('$type'))
+            sb.add_line('@type', v)
+          end
+
+          if (v = config.delete('$label'))
+            sb.add_line('@label', v)
+          end
+
+          if (v = config.delete('$id'))
+            sb.add_line('@id', v)
+          end
+
+          if (v = config.delete('$log_level'))
+            sb.add_line('@log_level', v)
+          end
+
+          config.each do |key, val|
+            if val.is_a?(Array)
+              if section?(val.first)
+                val.each do |v|
+                  sb.add_section(section_build(key, v, indent: indent + @base_indent))
+                end
+              else
+                sb.add_line(key, val)
+              end
+            elsif val.is_a?(Hash)
+              harg = val.delete('$arg')
+              if harg.is_a?(Array)
+                # To prevent to generate invalid configuration for arg.
+                # "arg" should be String object and concatenated by ","
+                # when two or more objects are specified there.
+                sb.add_section(section_build(key, val, indent: indent + @base_indent, arg: harg&.join(',')))
+              else
+                sb.add_section(section_build(key, val, indent: indent + @base_indent, arg: harg))
+              end
+            else
+              sb.add_line(key, val)
+            end
+          end
+
+          SectionBuilder.new(name, sb, indent, arg)
+        end
+
+        def section?(value)
+          value.is_a?(Array) or value.is_a?(Hash)
+        end
+      end
+    end
+  end
+end

data/lib/fluent/config/yaml_parser/section_builder.rb

@@ -0,0 +1,107 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Config
+    module YamlParser
+      SectionBuilder = Struct.new(:name, :body, :indent_size, :arg) do
+        def to_s
+          indent = ' ' * indent_size
+
+          if arg && !arg.to_s.empty?
+            "#{indent}<#{name} #{arg}>\n#{body}\n#{indent}</#{name}>"
+          else
+            "#{indent}<#{name}>\n#{body}\n#{indent}</#{name}>"
+          end
+        end
+
+        def to_element
+          elem = body.to_element
+          elem.name = name
+          elem.arg = arg.to_s if arg
+          elem.v1_config = true
+          elem
+        end
+      end
+
+      class RootBuilder
+        def initialize(system, conf)
+          @system = system
+          @conf = conf
+        end
+
+        attr_reader :system, :conf
+
+        def to_element
+          Fluent::Config::Element.new('ROOT', '', {}, [@system, @conf].compact.map(&:to_element).flatten)
+        end
+
+        def to_s
+          s = StringIO.new(+'')
+          s.puts(@system.to_s) if @system
+          s.puts(@conf.to_s) if @conf
+
+          s.string
+        end
+      end
+
+      class SectionBodyBuilder
+        Row = Struct.new(:key, :value, :indent) do
+          def to_s
+            "#{indent}#{key} #{value}"
+          end
+        end
+
+        def initialize(indent, root: false)
+          @indent = ' ' * indent
+          @bodies = []
+          @root = root
+        end
+
+        def add_line(k, v)
+          @bodies << Row.new(k, v, @indent)
+        end
+
+        def add_section(section)
+          @bodies << section
+        end
+
+        def to_element
+          if @root
+            return @bodies.map(&:to_element)
+          end
+
+          not_section, section = @bodies.partition { |e| e.is_a?(Row) }
+          r = {}
+          not_section.each do |e|
+            v = e.value
+            r[e.key] = v.respond_to?(:to_element) ? v.to_element : v
+          end
+
+          if @root
+            section.map(&:to_element)
+          else
+            Fluent::Config::Element.new('', '', r, section.map(&:to_element))
+          end
+        end
+
+        def to_s
+          @bodies.map(&:to_s).join("\n")
+        end
+      end
+    end
+  end
+end