fluentd-hubspot 0.14.14.1

data/lib/fluent/config/dsl.rb

@@ -0,0 +1,149 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'json'
+
+require 'fluent/config'
+require 'fluent/config/element'
+
+module Fluent
+  module Config
+    module DSL
+      module Parser
+        def self.read(path)
+          path = File.expand_path(path)
+          data = File.read(path)
+          parse(data, path)
+        end
+
+        def self.parse(source, source_path="config.rb")
+          Proxy.new('ROOT', nil).eval(source, source_path).to_config_element
+        end
+      end
+
+      class Proxy
+        def initialize(name, arg, include_basepath = Dir.pwd)
+          @element = Element.new(name, arg, self)
+          @include_basepath = include_basepath
+        end
+
+        def element
+          @element
+        end
+
+        def include_basepath
+          @include_basepath
+        end
+
+        def eval(source, source_path)
+          @element.instance_eval(source, source_path)
+          self
+        end
+
+        def to_config_element
+          @element.instance_eval do
+            Config::Element.new(@name, @arg, @attrs, @elements)
+          end
+        end
+
+        def add_element(name, arg, block)
+          ::Kernel.raise ::ArgumentError, "#{name} block must be specified" if block.nil?
+
+          proxy = self.class.new(name.to_s, arg)
+          proxy.element.instance_exec(&block)
+
+          @element.instance_eval do
+            @elements.push(proxy.to_config_element)
+          end
+
+          self
+        end
+      end
+
+      class Element < BasicObject
+        def initialize(name, arg, proxy)
+          @name     = name
+          @arg      = arg || ''
+          @attrs    = {}
+          @elements = []
+          @proxy    = proxy
+        end
+
+        def to_int
+          __id__
+        end
+
+        def method_missing(name, *args, &block)
+          ::Kernel.raise ::ArgumentError, "Configuration DSL Syntax Error: only one argument allowed" if args.size > 1
+          value = args.first
+
+          if block
+            proxy = Proxy.new(name.to_s, value)
+            proxy.element.instance_exec(&block)
+            @elements.push(proxy.to_config_element)
+          else
+            @attrs[name.to_s] = if value.is_a?(Array) || value.is_a?(Hash)
+                                  JSON.dump(value)
+                                else
+                                  value.to_s
+                                end
+          end
+
+          self
+        end
+
+        def include(*args)
+          ::Kernel.raise ::ArgumentError, "#{name} block requires arguments for include path" if args.nil? || args.size != 1
+          if args.first =~ /\.rb$/
+            path = File.expand_path(args.first)
+            data = File.read(path)
+            self.instance_eval(data, path)
+          else
+            ss = StringScanner.new('')
+            Config::V1Parser.new(ss, @proxy.include_basepath, '', nil).eval_include(@attrs, @elements, args.first)
+          end
+        end
+
+        def source(&block)
+          @proxy.add_element('source', nil, block)
+        end
+
+        def match(*args, &block)
+          ::Kernel.raise ::ArgumentError, "#{name} block requires arguments for match pattern" if args.nil? || args.size != 1
+          @proxy.add_element('match', args.first, block)
+        end
+
+        def self.const_missing(name)
+          return ::Kernel.const_get(name) if ::Kernel.const_defined?(name)
+
+          if name.to_s =~ /^Fluent::Config::DSL::Element::(.*)$/
+            name = "#{$1}".to_sym
+            return ::Kernel.const_get(name) if ::Kernel.const_defined?(name)
+          end
+          ::Kernel.eval("#{name}")
+        end
+
+        def ruby(&block)
+          if block
+            @proxy.instance_exec(&block)
+          else
+            ::Kernel
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/config/element.rb

@@ -0,0 +1,218 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config/error'
+require 'fluent/config/literal_parser'
+
+module Fluent
+  module Config
+    class Element < Hash
+      def initialize(name, arg, attrs, elements, unused = nil)
+        @name = name
+        @arg = arg
+        @elements = elements
+        super()
+        attrs.each { |k, v|
+          self[k] = v
+        }
+        @unused = unused || attrs.keys
+        @v1_config = false
+        @corresponding_proxies = [] # some plugins use flat parameters, e.g. in_http doesn't provide <format> section for parser.
+        @unused_in = false # if this element is not used in plugins, correspoing plugin name and parent element name is set, e.g. [source, plugin class].
+
+        # it's global logger, not plugin logger: deprecated message should be global warning, not plugin level.
+        @logger = defined?($log) ? $log : nil
+      end
+
+      attr_accessor :name, :arg, :unused, :v1_config, :corresponding_proxies, :unused_in
+      attr_writer :elements
+
+      RESERVED_PARAMETERS_COMPAT = {
+        '@type' => 'type',
+        '@id' => 'id',
+        '@log_level' => 'log_level',
+        '@label' => nil,
+      }
+      RESERVED_PARAMETERS = RESERVED_PARAMETERS_COMPAT.keys
+
+      def elements(*names, name: nil, arg: nil)
+        raise ArgumentError, "name and names are exclusive" if name && !names.empty?
+        raise ArgumentError, "arg is available only with name" if arg && !name
+
+        if name
+          @elements.select{|e| e.name == name && (!arg || e.arg == arg) }
+        elsif !names.empty?
+          @elements.select{|e| names.include?(e.name) }
+        else
+          @elements
+        end
+      end
+
+      def add_element(name, arg = '')
+        e = Element.new(name, arg, {}, [])
+        e.v1_config = @v1_config
+        @elements << e
+        e
+      end
+
+      def inspect
+        attrs = super
+        "name:#{@name}, arg:#{@arg}, " + attrs + ", " + @elements.inspect
+      end
+
+      # This method assumes _o_ is an Element object. Should return false for nil or other object
+      def ==(o)
+        self.name == o.name && self.arg == o.arg &&
+          self.keys.size == o.keys.size &&
+          self.keys.reduce(true){|r, k| r && self[k] == o[k] } &&
+          self.elements.size == o.elements.size &&
+          [self.elements, o.elements].transpose.reduce(true){|r, e| r && e[0] == e[1] }
+      end
+
+      def +(o)
+        e = Element.new(@name.dup, @arg.dup, o.merge(self), @elements + o.elements, (@unused + o.unused).uniq)
+        e.v1_config = @v1_config
+        e
+      end
+
+      # no code in fluentd uses this method
+      def each_element(*names, &block)
+        if names.empty?
+          @elements.each(&block)
+        else
+          @elements.each { |e|
+            if names.include?(e.name)
+              block.yield(e)
+            end
+          }
+        end
+      end
+
+      def has_key?(key)
+        @unused_in = false # some sections, e.g. <store> in copy, is not defined by config_section so clear unused flag for better warning message in check_not_fetched.
+        @unused.delete(key)
+        super
+      end
+
+      def [](key)
+        @unused_in = false # ditto
+        @unused.delete(key)
+
+        if RESERVED_PARAMETERS.include?(key) && !has_key?(key) && has_key?(RESERVED_PARAMETERS_COMPAT[key])
+          @logger.warn "'#{RESERVED_PARAMETERS_COMPAT[key]}' is deprecated parameter name. use '#{key}' instead." if @logger
+          return self[RESERVED_PARAMETERS_COMPAT[key]]
+        end
+
+        super
+      end
+
+      def check_not_fetched(&block)
+        each_key { |key|
+          if @unused.include?(key)
+            block.call(key, self)
+          end
+        }
+        @elements.each { |e|
+          e.check_not_fetched(&block)
+        }
+      end
+
+      def to_s(nest = 0)
+        indent = "  " * nest
+        nindent = "  " * (nest + 1)
+        out = ""
+        if @arg.empty?
+          out << "#{indent}<#{@name}>\n"
+        else
+          out << "#{indent}<#{@name} #{@arg}>\n"
+        end
+        each_pair { |k, v|
+          out << dump_value(k, v, nindent)
+        }
+        @elements.each { |e|
+          out << e.to_s(nest + 1)
+        }
+        out << "#{indent}</#{@name}>\n"
+        out
+      end
+
+      def to_masked_element
+        new_elems = @elements.map { |e| e.to_masked_element }
+        new_elem = Element.new(@name, @arg, {}, new_elems, @unused)
+        new_elem.v1_config = @v1_config
+        new_elem.corresponding_proxies = @corresponding_proxies
+        each_pair { |k, v|
+          new_elem[k] = secret_param?(k) ? 'xxxxxx' : v
+        }
+        new_elem
+      end
+
+      def secret_param?(key)
+        return false if @corresponding_proxies.empty?
+
+        param_key = key.to_sym
+        @corresponding_proxies.each { |proxy|
+          _block, opts = proxy.params[param_key]
+          if opts && opts.has_key?(:secret)
+            return opts[:secret]
+          end
+        }
+
+        false
+      end
+
+      def param_type(key)
+        return nil if @corresponding_proxies.empty?
+
+        param_key = key.to_sym
+        proxy = @corresponding_proxies.detect do |_proxy|
+          _proxy.params.has_key?(param_key)
+        end
+        return nil unless proxy
+        _block, opts = proxy.params[param_key]
+        opts[:type]
+      end
+
+      def dump_value(k, v, nindent)
+        if secret_param?(k)
+          "#{nindent}#{k} xxxxxx\n"
+        else
+          if @v1_config
+            case param_type(k)
+            when :string
+              "#{nindent}#{k} \"#{self.class.unescape_parameter(v)}\"\n"
+            when :enum, :integer, :float, :size, :bool, :time
+              "#{nindent}#{k} #{v}\n"
+            when :hash, :array
+              "#{nindent}#{k} #{v}\n"
+            else
+              # Unknown type
+              "#{nindent}#{k} #{v}\n"
+            end
+          else
+            "#{nindent}#{k} #{v}\n"
+          end
+        end
+      end
+
+      def self.unescape_parameter(v)
+        result = ''
+        v.each_char { |c| result << LiteralParser.unescape_char(c) }
+        result
+      end
+    end
+  end
+end

data/lib/fluent/config/error.rb

@@ -0,0 +1,26 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  class ConfigError < StandardError
+  end
+
+  class ConfigParseError < ConfigError
+  end
+
+  class ObsoletedParameterError < ConfigError
+  end
+end

data/lib/fluent/config/literal_parser.rb

@@ -0,0 +1,251 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'stringio'
+
+require 'json'
+require 'yajl'
+require 'irb/ruby-lex'  # RubyLex
+
+require 'fluent/config/basic_parser'
+
+module Fluent
+  module Config
+    class LiteralParser < BasicParser
+      def self.unescape_char(c)
+        case c
+        when '"'
+          '\"'
+        when "'"
+          "\\'"
+        when '\\'
+          '\\\\'
+        when "\r"
+          '\r'
+        when "\n"
+          '\n'
+        when "\t"
+          '\t'
+        when "\f"
+          '\f'
+        when "\b"
+          '\b'
+        else
+          c
+        end
+      end
+
+      def initialize(strscan, eval_context)
+        super(strscan)
+        @eval_context = eval_context
+      end
+
+      def parse_literal(string_boundary_charset = LINE_END)
+        spacing_without_comment
+
+        value = if skip(/\[/)
+                  scan_json(true)
+                elsif skip(/\{/)
+                  scan_json(false)
+                else
+                  scan_string(string_boundary_charset)
+                end
+        value
+      end
+
+      def scan_string(string_boundary_charset = LINE_END)
+        if skip(/\"/)
+          return scan_double_quoted_string
+        elsif skip(/\'/)
+          return scan_single_quoted_string
+        else
+          return scan_nonquoted_string(string_boundary_charset)
+        end
+      end
+
+      def scan_double_quoted_string
+        string = []
+        while true
+          if skip(/\"/)
+            return string.join
+          elsif check(/[^"]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/)
+            if s = check(/[^\\]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/)
+              string << s
+            end
+            skip(/[^"]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/)
+          elsif s = scan(/\\./)
+            string << eval_escape_char(s[1,1])
+          elsif skip(/\#\{/)
+            string << eval_embedded_code(scan_embedded_code)
+            skip(/\}/)
+          elsif s = scan(/./)
+            string << s
+          else
+            parse_error! "unexpected end of file in a double quoted string"
+          end
+        end
+      end
+
+      def scan_single_quoted_string
+        string = []
+        while true
+          if skip(/\'/)
+            return string.join
+          elsif s = scan(/\\'/)
+            string << "'"
+          elsif s = scan(/\\\\/)
+            string << "\\"
+          elsif s = scan(/./)
+            string << s
+          else
+            parse_error! "unexpected end of file in a signle quoted string"
+          end
+        end
+      end
+
+      def scan_nonquoted_string(boundary_charset = LINE_END)
+        charset = /(?!#{boundary_charset})./
+
+        string = []
+        while true
+          if s = scan(/\#/)
+            string << '#'
+          elsif s = scan(charset)
+            string << s
+          else
+            break
+          end
+        end
+
+        if string.empty?
+          return nil
+        end
+
+        string.join
+      end
+
+      def scan_embedded_code
+        rlex = RubyLex.new
+        src = '"#{'+@ss.rest+"\n=end\n}"
+
+        input = StringIO.new(src)
+        input.define_singleton_method(:encoding) { external_encoding }
+        rlex.set_input(input)
+
+        tk = rlex.token
+        code = src[3,tk.seek-3]
+
+        if @ss.rest.length < code.length
+          @ss.pos += @ss.rest.length
+          parse_error! "expected end of embedded code but $end"
+        end
+
+        @ss.pos += code.length
+
+        '"#{' + code + '}"'
+      end
+
+      def eval_embedded_code(code)
+        if @eval_context.nil?
+          parse_error! "embedded code is not allowed in this file"
+        end
+        @eval_context.instance_eval(code)
+      end
+
+      def eval_escape_char(c)
+        case c
+        when '"'
+          '"'
+        when "'"
+          "'"
+        when "r"
+          "\r"
+        when "n"
+          "\n"
+        when "t"
+          "\t"
+        when "f"
+          "\f"
+        when "b"
+          "\b"
+        when /[a-zA-Z0-9]/
+          parse_error! "unexpected back-slash escape character '#{c}'"
+        else  # symbols
+          c
+        end
+      end
+
+      def scan_json(is_array)
+        result = nil
+        # Yajl does not raise ParseError for imcomplete json string, like '[1', '{"h"', '{"h":' or '{"h1":1'
+        # This is the reason to use JSON module.
+
+        buffer = (is_array ? "[" : "{")
+        line_buffer = ""
+
+        until result
+          char = getch
+
+          break if char.nil?
+
+          if char == "#"
+            # If this is out of json string literals, this object can be parsed correctly
+            # '{"foo":"bar", #' -> '{"foo":"bar"}' (to check)
+            parsed = nil
+            begin
+              parsed = JSON.parse(buffer + line_buffer.rstrip.sub(/,$/, '') + (is_array ? "]" : "}"))
+            rescue JSON::ParserError => e
+              # This '#' is in json string literals
+            end
+
+            if parsed
+              # ignore chars as comment before newline
+              while (char = getch) != "\n"
+                # ignore comment char
+              end
+              buffer << line_buffer + "\n"
+              line_buffer = ""
+            else
+              # '#' is a char in json string
+              line_buffer << char
+            end
+
+            next # This char '#' MUST NOT terminate json object.
+          end
+
+          if char == "\n"
+            buffer << line_buffer + "\n"
+            line_buffer = ""
+            next
+          end
+
+          line_buffer << char
+          begin
+            result = JSON.parse(buffer + line_buffer)
+          rescue JSON::ParserError => e
+            # Incomplete json string yet
+          end
+        end
+
+        unless result
+          parse_error! "got incomplete JSON #{is_array ? 'array' : 'hash'} configuration"
+        end
+
+        JSON.dump(result)
+      end
+    end
+  end
+end