fluentd-hubspot 0.14.14.1

data/lib/fluent/config/parser.rb

@@ -0,0 +1,107 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'uri'
+
+require 'fluent/config/error'
+require 'fluent/config/element'
+
+module Fluent
+  module Config
+    class Parser
+      def self.parse(io, fname, basepath = Dir.pwd)
+        attrs, elems = Parser.new(basepath, io.each_line, fname).parse!(true)
+        Element.new('ROOT', '', attrs, elems)
+      end
+
+      def initialize(basepath, iterator, fname, i = 0)
+        @basepath = basepath
+        @iterator = iterator
+        @i = i
+        @fname = fname
+      end
+
+      def parse!(allow_include, elem_name = nil, attrs = {}, elems = [])
+        while line = @iterator.next
+          line.force_encoding('UTF-8')
+          @i += 1
+          line.lstrip!
+          line.gsub!(/\s*(?:\#.*)?$/,'')
+          if line.empty?
+            next
+          elsif m = /^\<include\s*(.*)\s*\/\>$/.match(line)
+            value = m[1].strip
+            process_include(attrs, elems, value, allow_include)
+          elsif m = /^\<([a-zA-Z0-9_]+)\s*(.+?)?\>$/.match(line)
+            e_name = m[1]
+            e_arg = m[2] || ""
+            e_attrs, e_elems = parse!(false, e_name)
+            elems << Element.new(e_name, e_arg, e_attrs, e_elems)
+          elsif line == "</#{elem_name}>"
+            break
+          elsif m = /^([a-zA-Z0-9_]+)\s*(.*)$/.match(line)
+            key = m[1]
+            value = m[2]
+            if allow_include && key == 'include'
+              process_include(attrs, elems, value)
+            else
+              attrs[key] = value
+            end
+            next
+          else
+            raise ConfigParseError, "parse error at #{@fname} line #{@i}"
+          end
+        end
+
+        return attrs, elems
+      rescue StopIteration
+        return attrs, elems
+      end
+
+      def process_include(attrs, elems, uri, allow_include = true)
+        u = URI.parse(uri)
+        if u.scheme == 'file' || u.path == uri  # file path
+          path = u.path
+          if path[0] != ?/
+            pattern = File.expand_path("#{@basepath}/#{path}")
+          else
+            pattern = path
+          end
+
+          Dir.glob(pattern).sort.each { |entry|
+            basepath = File.dirname(entry)
+            fname = File.basename(entry)
+            File.open(entry) { |f|
+              Parser.new(basepath, f.each_line, fname).parse!(allow_include, nil, attrs, elems)
+            }
+          }
+
+        else
+          basepath = '/'
+          fname = path
+          require 'open-uri'
+          open(uri) {|f|
+            Parser.new(basepath, f.each_line, fname).parse!(allow_include, nil, attrs, elems)
+          }
+        end
+
+      rescue SystemCallError => e
+        raise ConfigParseError, "include error at #{@fname} line #{@i}: #{e.to_s}"
+      end
+    end
+  end
+end
+

data/lib/fluent/config/section.rb

@@ -0,0 +1,223 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'json'
+
+require 'fluent/config/error'
+require 'fluent/config/v1_parser'
+
+module Fluent
+  module Config
+    class Section < BasicObject
+      def self.name
+        'Fluent::Config::Section'
+      end
+
+      def initialize(params = {}, config_element = nil)
+        @klass = 'Fluent::Config::Section'
+        @params = params
+        @corresponding_config_element = config_element
+      end
+
+      alias :object_id :__id__
+
+      def corresponding_config_element
+        @corresponding_config_element
+      end
+
+      def class
+        Section
+      end
+
+      def to_s
+        inspect
+      end
+
+      def inspect
+        "<Fluent::Config::Section #{@params.to_json}>"
+      end
+
+      def nil?
+        false
+      end
+
+      def to_h
+        @params
+      end
+
+      def +(other)
+        Section.new(self.to_h.merge(other.to_h))
+      end
+
+      def instance_of?(mod)
+        @klass == mod.name
+      end
+
+      def kind_of?(mod)
+        @klass == mod.name || BasicObject == mod
+      end
+      alias is_a? kind_of?
+
+      def [](key)
+        @params[key.to_sym]
+      end
+
+      def []=(key, value)
+        @params[key.to_sym] = value
+      end
+
+      def respond_to?(symbol, include_all=false)
+        case symbol
+        when :inspect, :nil?, :to_h, :+, :instance_of?, :kind_of?, :[], :respond_to?, :respond_to_missing?
+          true
+        when :!, :!= , :==, :equal?, :instance_eval, :instance_exec
+          true
+        when :method_missing, :singleton_method_added, :singleton_method_removed, :singleton_method_undefined
+          include_all
+        else
+          false
+        end
+      end
+
+      def respond_to_missing?(symbol, include_private)
+        @params.has_key?(symbol)
+      end
+
+      def method_missing(name, *args)
+        if @params.has_key?(name)
+          @params[name]
+        else
+          ::Kernel.raise ::NoMethodError, "undefined method `#{name}' for #{self.inspect}"
+        end
+      end
+    end
+
+    module SectionGenerator
+      def self.generate(proxy, conf, logger, plugin_class, stack = [])
+        return nil if conf.nil?
+
+        section_stack = ""
+        unless stack.empty?
+          section_stack = ", in section " + stack.join(" > ")
+        end
+
+        section_params = {}
+
+        proxy.defaults.each_pair do |name, defval|
+          varname = name.to_sym
+          section_params[varname] = (defval.dup rescue defval)
+        end
+
+        if proxy.argument
+          unless conf.arg.empty?
+            key, block, opts = proxy.argument
+            section_params[key] = self.instance_exec(conf.arg, opts, name, &block)
+          end
+          unless section_params.has_key?(proxy.argument.first)
+            logger.error "config error in:\n#{conf}" if logger # logger should exist, but somethimes it's nil (e.g, in tests)
+            raise ConfigError, "'<#{proxy.name} ARG>' section requires argument" + section_stack
+          end
+          # argument should NOT be deprecated... (argument always has a value: '')
+        end
+
+        proxy.params.each_pair do |name, defval|
+          varname = name.to_sym
+          block, opts = defval
+          if conf.has_key?(name.to_s) || opts[:alias] && conf.has_key?(opts[:alias].to_s)
+            val = if conf.has_key?(name.to_s)
+                    conf[name.to_s]
+                  else
+                    conf[opts[:alias].to_s]
+                  end
+            section_params[varname] = self.instance_exec(val, opts, name, &block)
+
+            # Source of definitions of deprecated/obsoleted:
+            # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Deprecated_and_obsolete_features
+            #
+            # Deprecated: These deprecated features can still be used, but should be used with caution
+            #             because they are expected to be removed entirely sometime in the future.
+            # Obsoleted: These obsolete features have been entirely removed from JavaScript and can no longer be used.
+            if opts[:deprecated]
+              logger.warn "'#{name}' parameter is deprecated: #{opts[:deprecated]}" if logger
+            end
+            if opts[:obsoleted]
+              logger.error "config error in:\n#{conf}" if logger
+              raise ObsoletedParameterError, "'#{name}' parameter is already removed: #{opts[:obsoleted]}" + section_stack
+            end
+          end
+          unless section_params.has_key?(varname)
+            logger.error "config error in:\n#{conf}" if logger
+            raise ConfigError, "'#{name}' parameter is required" + section_stack
+          end
+        end
+
+        check_unused_section(proxy, conf, plugin_class)
+
+        proxy.sections.each do |name, subproxy|
+          varname = subproxy.variable_name
+          elements = (conf.respond_to?(:elements) ? conf.elements : []).select{ |e| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }
+          if elements.empty? && subproxy.init?
+            if subproxy.argument && !subproxy.defaults.has_key?(subproxy.argument.first)
+              raise ArgumentError, "#{name}: init is specified, but default value of argument is missing"
+            end
+            missing_keys = subproxy.params.keys.select{|param_name| !subproxy.defaults.has_key?(param_name)}
+            if !missing_keys.empty?
+              raise ArgumentError, "#{name}: init is specified, but there're parameters without default values:#{missing_keys.join(',')}"
+            end
+            elements << Fluent::Config::Element.new(subproxy.name.to_s, '', {}, [])
+          end
+
+          # set subproxy for secret option
+          elements.each { |element|
+            element.corresponding_proxies << subproxy
+          }
+
+          if subproxy.required? && elements.size < 1
+            logger.error "config error in:\n#{conf}" if logger
+            raise ConfigError, "'<#{subproxy.name}>' sections are required" + section_stack
+          end
+          if subproxy.multi?
+            section_params[varname] = elements.map{ |e| generate(subproxy, e, logger, plugin_class, stack + [subproxy.name]) }
+          else
+            if elements.size > 1
+              logger.error "config error in:\n#{conf}" if logger
+              raise ConfigError, "'<#{subproxy.name}>' section cannot be written twice or more" + section_stack
+            end
+            section_params[varname] = generate(subproxy, elements.first, logger, plugin_class, stack + [subproxy.name])
+          end
+        end
+
+        Section.new(section_params, conf)
+      end
+
+      def self.check_unused_section(proxy, conf, plugin_class)
+        elems = conf.respond_to?(:elements) ? conf.elements : []
+        elems.each { |e|
+          next if plugin_class.nil? && Fluent::Config::V1Parser::ELEM_SYMBOLS.include?(e.name) # skip pre-defined non-plugin elements because it doens't have proxy section
+
+          unless proxy.sections.any? { |name, subproxy| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }
+            parent_name = if conf.arg.empty?
+                            conf.name
+                          else
+                            "#{conf.name} #{conf.arg}"
+                          end
+            e.unused_in = [parent_name, plugin_class]
+          end
+        }
+      end
+    end
+  end
+end

data/lib/fluent/config/types.rb

@@ -0,0 +1,136 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'json'
+
+require 'fluent/config/error'
+
+module Fluent
+  module Config
+    def self.size_value(str)
+      case str.to_s
+      when /([0-9]+)k/i
+        $~[1].to_i * 1024
+      when /([0-9]+)m/i
+        $~[1].to_i * (1024 ** 2)
+      when /([0-9]+)g/i
+        $~[1].to_i * (1024 ** 3)
+      when /([0-9]+)t/i
+        $~[1].to_i * (1024 ** 4)
+      else
+        str.to_i
+      end
+    end
+
+    def self.time_value(str)
+      case str.to_s
+      when /([0-9]+)s/
+        $~[1].to_i
+      when /([0-9]+)m/
+        $~[1].to_i * 60
+      when /([0-9]+)h/
+        $~[1].to_i * 60 * 60
+      when /([0-9]+)d/
+        $~[1].to_i * 24 * 60 * 60
+      else
+        str.to_f
+      end
+    end
+
+    def self.bool_value(str)
+      return nil if str.nil?
+      case str.to_s
+      when 'true', 'yes'
+        true
+      when 'false', 'no'
+        false
+      when ''
+        true
+      else
+        nil
+      end
+    end
+
+    STRING_TYPE = Proc.new { |val, opts| val.to_s.force_encoding(Encoding::UTF_8) }
+    ENUM_TYPE = Proc.new { |val, opts|
+      s = val.to_sym
+      list = opts[:list]
+      raise "Plugin BUG: config type 'enum' requires :list of symbols" unless list.is_a?(Array) && list.all?{|v| v.is_a? Symbol }
+      unless list.include?(s)
+        raise ConfigError, "valid options are #{list.join(',')} but got #{val}"
+      end
+      s
+    }
+    INTEGER_TYPE = Proc.new { |val, opts| val.to_i }
+    FLOAT_TYPE = Proc.new { |val, opts| val.to_f }
+    SIZE_TYPE = Proc.new { |val, opts| Config.size_value(val) }
+    BOOL_TYPE = Proc.new { |val, opts| Config.bool_value(val) }
+    TIME_TYPE = Proc.new { |val, opts| Config.time_value(val) }
+
+    REFORMAT_VALUE = ->(type, value) {
+      if value.nil?
+        value
+      else
+        case type
+        when :string  then value.to_s.force_encoding(Encoding::UTF_8)
+        when :integer then value.to_i
+        when :float   then value.to_f
+        when :size then Config.size_value(value)
+        when :bool then Config.bool_value(value)
+        when :time then Config.time_value(value)
+        else
+          raise "unknown type in REFORMAT: #{type}"
+        end
+      end
+    }
+
+    HASH_TYPE = Proc.new { |val, opts|
+      param = if val.is_a?(String)
+                val.start_with?('{') ? JSON.load(val) : Hash[val.strip.split(/\s*,\s*/).map{|v| v.split(':', 2)}]
+              else
+                val
+              end
+      if param.class != Hash
+        raise ConfigError, "hash required but got #{val.inspect}"
+      end
+      if opts.empty?
+        param
+      else
+        newparam = {}
+        param.each_pair do |key, value|
+          new_key = opts[:symbolize_keys] ? key.to_sym : key
+          newparam[new_key] = opts[:value_type] ? REFORMAT_VALUE.call(opts[:value_type], value) : value
+        end
+        newparam
+      end
+    }
+    ARRAY_TYPE = Proc.new { |val, opts|
+      param = if val.is_a?(String)
+                val.start_with?('[') ? JSON.load(val) : val.strip.split(/\s*,\s*/)
+              else
+                val
+              end
+      if param.class != Array
+        raise ConfigError, "array required but got #{val.inspect}"
+      end
+      if opts[:value_type]
+        param.map{|v| REFORMAT_VALUE.call(opts[:value_type], v) }
+      else
+        param
+      end
+    }
+  end
+end