fluentd 0.10.45 → 0.10.46

data/lib/fluent/config/parser.rb

@@ -0,0 +1,96 @@
+module Fluent
+  module Config
+    require 'fluent/config/error'
+    require 'fluent/config/element'
+
+    class Parser
+      def self.read(path)
+        path = File.expand_path(path)
+        File.open(path) { |io|
+          parse(io, File.basename(path), File.dirname(path))
+        }
+      end
+
+      def self.parse(io, fname, basepath = Dir.pwd)
+        attrs, elems = Parser.new(basepath, io.each_line, fname).parse!(true)
+        Element.new('ROOT', '', attrs, elems)
+      end
+
+      def initialize(basepath, iterator, fname, i = 0)
+        @basepath = basepath
+        @iterator = iterator
+        @i = i
+        @fname = fname
+      end
+
+      def parse!(allow_include, elem_name = nil, attrs = {}, elems = [])
+        while line = @iterator.next
+          line.force_encoding('UTF-8')
+          @i += 1
+          line.lstrip!
+          line.gsub!(/\s*(?:\#.*)?$/,'')
+          if line.empty?
+            next
+          elsif m = /^\<include\s*(.*)\s*\/\>$/.match(line)
+            value = m[1].strip
+            process_include(attrs, elems, value, allow_include)
+          elsif m = /^\<([a-zA-Z0-9_]+)\s*(.+?)?\>$/.match(line)
+            e_name = m[1]
+            e_arg = m[2] || ""
+            e_attrs, e_elems = parse!(false, e_name)
+            elems << Element.new(e_name, e_arg, e_attrs, e_elems)
+          elsif line == "</#{elem_name}>"
+            break
+          elsif m = /^([a-zA-Z0-9_]+)\s*(.*)$/.match(line)
+            key = m[1]
+            value = m[2]
+            if allow_include && key == 'include'
+              process_include(attrs, elems, value)
+            else
+              attrs[key] = value
+            end
+            next
+          else
+            raise ConfigParseError, "parse error at #{@fname} line #{@i}"
+          end
+        end
+
+        return attrs, elems
+      rescue StopIteration
+        return attrs, elems
+      end
+
+      def process_include(attrs, elems, uri, allow_include = true)
+        u = URI.parse(uri)
+        if u.scheme == 'file' || u.path == uri  # file path
+          path = u.path
+          if path[0] != ?/
+            pattern = File.expand_path("#{@basepath}/#{path}")
+          else
+            pattern = path
+          end
+
+          Dir.glob(pattern).sort.each { |path|
+            basepath = File.dirname(path)
+            fname = File.basename(path)
+            File.open(path) { |f|
+              Parser.new(basepath, f.each_line, fname).parse!(allow_include, nil, attrs, elems)
+            }
+          }
+
+        else
+          basepath = '/'
+          fname = path
+          require 'open-uri'
+          open(uri) {|f|
+            Parser.new(basepath, f.each_line, fname).parse!(allow_include, nil, attrs, elems)
+          }
+        end
+
+      rescue SystemCallError => e
+        raise ConfigParseError, "include error at #{@fname} line #{@i}: #{e.to_s}"
+      end
+    end
+  end
+end
+

data/lib/fluent/config/section.rb

@@ -0,0 +1,115 @@
+module Fluent
+  require 'fluent/config/error'
+
+  module Config
+    class Section < BasicObject
+      def self.name
+        'Fluent::Config::Section'
+      end
+
+      def initialize(params = {})
+        @klass = 'Fluent::Config::Section'
+        @params = params
+      end
+
+      def nil?
+        false
+      end
+
+      def to_h
+        @params
+      end
+
+      def +(other)
+        Section.new(self.to_h.merge(other.to_h))
+      end
+
+      def instance_of?(mod)
+        @klass == mod.name
+      end
+
+      def kind_of?(mod)
+        @klass == mod.name || BasicObject == mod
+      end
+      alias is_a? kind_of?
+
+      def [](key)
+        @params[key.to_sym]
+      end
+
+      def respond_to_missing?(symbol, include_private)
+        @params.has_key?(symbol)
+      end
+
+      def method_missing(name, *args)
+        if @params.has_key?(name)
+          @params[name]
+        else
+          super
+        end
+      end
+    end
+
+    module SectionGenerator
+      def self.generate(proxy, conf, logger, stack = [])
+        return nil if conf.nil?
+
+        section_stack = ""
+        unless stack.empty?
+          section_stack = ", in section " + stack.join(" > ")
+        end
+
+        section_params = {}
+
+        proxy.defaults.each_pair do |name, defval|
+          varname = name.to_sym
+          section_params[varname] = defval
+        end
+
+        if proxy.argument
+          unless conf.arg.empty?
+            key, block, opts = proxy.argument
+            section_params[key] = self.instance_exec(conf.arg, opts, name, &block)
+          end
+          unless section_params.has_key?(proxy.argument.first)
+            logger.error "config error in:\n#{conf}"
+            raise ConfigError, "'<#{proxy.name} ARG>' section requires argument" + section_stack
+          end
+        end
+
+        proxy.params.each_pair do |name, defval|
+          varname = name.to_sym
+          block, opts = defval
+          if val = conf[name.to_s]
+            section_params[varname] = self.instance_exec(val, opts, name, &block)
+          end
+          unless section_params.has_key?(varname)
+            logger.error "config error in:\n#{conf}"
+            raise ConfigError, "'#{name}' parameter is required" + section_stack
+          end
+        end
+
+        proxy.sections.each do |name, subproxy|
+          varname = subproxy.param_name.to_sym
+          elements = (conf.respond_to?(:elements) ? conf.elements : []).select{ |e| e.name == subproxy.name.to_s }
+
+          if subproxy.required? && elements.size < 1
+            logger.error "config error in:\n#{conf}"
+            raise ConfigError, "'<#{subproxy.name}>' sections are required" + section_stack
+          end
+          if subproxy.multi?
+            section_params[varname] = elements.map{ |e| generate(subproxy, e, logger, stack + [subproxy.name]) }
+          else
+            if elements.size > 1
+              logger.error "config error in:\n#{conf}"
+              raise ConfigError, "'<#{subproxy.name}>' section cannot be written twice or more" + section_stack
+            end
+            section_params[varname] = generate(subproxy, elements.first, logger, stack + [subproxy.name])
+          end
+        end
+
+        Section.new(section_params)
+      end
+    end
+  end
+end

data/lib/fluent/config/types.rb

@@ -0,0 +1,86 @@
+module Fluent
+  require 'json'
+
+  module Config
+    def self.size_value(str)
+      case str.to_s
+      when /([0-9]+)k/i
+        $~[1].to_i * 1024
+      when /([0-9]+)m/i
+        $~[1].to_i * (1024 ** 2)
+      when /([0-9]+)g/i
+        $~[1].to_i * (1024 ** 3)
+      when /([0-9]+)t/i
+        $~[1].to_i * (1024 ** 4)
+      else
+        str.to_i
+      end
+    end
+
+    def self.time_value(str)
+      case str.to_s
+      when /([0-9]+)s/
+        $~[1].to_i
+      when /([0-9]+)m/
+        $~[1].to_i * 60
+      when /([0-9]+)h/
+        $~[1].to_i * 60 * 60
+      when /([0-9]+)d/
+        $~[1].to_i * 24 * 60 * 60
+      else
+        str.to_f
+      end
+    end
+
+    def self.bool_value(str)
+      case str.to_s
+      when 'true', 'yes'
+        true
+      when 'false', 'no'
+        false
+      else
+        nil
+      end
+    end
+  end
+
+  Configurable.register_type(:string, Proc.new { |val, opts|
+    val
+  })
+
+  Configurable.register_type(:integer, Proc.new { |val, opts|
+    val.to_i
+  })
+
+  Configurable.register_type(:float, Proc.new { |val, opts|
+    val.to_f
+  })
+
+  Configurable.register_type(:size, Proc.new { |val, opts|
+    Config.size_value(val)
+  })
+
+  Configurable.register_type(:bool, Proc.new { |val, opts|
+    Config.bool_value(val)
+  })
+
+  Configurable.register_type(:time, Proc.new { |val, opts|
+    Config.time_value(val)
+  })
+
+  Configurable.register_type(:hash, Proc.new { |val, opts|
+    param = JSON.load(val)
+    if param.class != Hash
+      raise ConfigError, "hash required but got #{val.inspect}"
+    end
+    param
+  })
+
+  Configurable.register_type(:array, Proc.new { |val, opts|
+    param = JSON.load(val)
+    if param.class != Array
+      raise ConfigError, "array required but got #{val.inspect}"
+    end
+    param
+  })
+end

data/lib/fluent/config/v1_parser.rb

@@ -0,0 +1,156 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+  module Config
+
+    require 'strscan'
+    require 'fluent/config/error'
+    require 'fluent/config/literal_parser'
+    require 'fluent/config/element'
+
+    class V1Parser < LiteralParser
+      ELEMENT_NAME = /[a-zA-Z0-9_]+/
+
+      def self.read(path, eval_context = nil)
+        path = File.expand_path(path)
+        data = File.read(path)
+        parse(data, File.basename(path), File.dirname(path), eval_context)
+      end
+
+      def self.parse(data, fname, basepath = Dir.pwd, eval_context = nil)
+        ss = StringScanner.new(data)
+        ps = V1Parser.new(ss, basepath, fname, eval_context)
+        ps.parse!
+      end
+
+      def initialize(strscan, include_basepath, fname, eval_context)
+        super(strscan, eval_context)
+        @include_basepath = include_basepath
+        @fname = fname
+      end
+
+      def parse!
+        attrs, elems = parse_element(true, nil)
+        root = Element.new('ROOT', '', attrs, elems)
+
+        spacing
+        unless eof?
+          parse_error! "expected EOF"
+        end
+
+        return root
+      end
+
+      def parse_element(root_element, elem_name, attrs = {}, elems = [])
+        while true
+          spacing
+          if eof?
+            if root_element
+              break
+            end
+            parse_error! "expected end tag '</#{elem_name}>' but got end of file"
+          end
+
+          if skip(/\<\//)
+            e_name = scan(ELEMENT_NAME)
+            spacing
+            unless skip(/\>/)
+              parse_error! "expected character in tag name"
+            end
+            unless line_end
+              parse_error! "expected end of line after end tag"
+            end
+            if e_name != elem_name
+              parse_error! "unmatched end tag"
+            end
+            break
+
+          elsif skip(/\</)
+            e_name = scan(ELEMENT_NAME)
+            spacing
+            e_arg = scan_nonquoted_string(/(?:#{SPACING}|\>)/)
+            spacing
+            unless skip(/\>/)
+              parse_error! "expected '>'"
+            end
+            unless line_end
+              parse_error! "expected end of line after tag"
+            end
+            e_arg ||= ''
+            # call parse_element recursively
+            e_attrs, e_elems = parse_element(false, e_name)
+            elems << Element.new(e_name, e_arg, e_attrs, e_elems)
+
+          elsif root_element && skip(/\@include#{SPACING}/)
+            uri = scan_string(LINE_END)
+            eval_include(attrs, elems, uri)
+            line_end
+
+          else
+            k = scan_string(SPACING)
+            spacing
+            v = parse_literal
+            unless line_end
+              parse_error! "expected end of line"
+            end
+            attrs[k] = v
+          end
+        end
+
+        return attrs, elems
+      end
+
+      def eval_include(attrs, elems, uri)
+        u = URI.parse(uri)
+        if u.scheme == 'file' || u.path == uri  # file path
+          path = u.path
+          if path[0] != ?/
+            pattern = File.expand_path("#{@include_basepath}/#{path}")
+          else
+            pattern = path
+          end
+
+          Dir.glob(pattern).each { |path|
+            basepath = File.dirname(path)
+            fname = File.basename(path)
+            data = File.read(path)
+            ss = StringScanner.new(data)
+            V1Parser.new(ss, basepath, fname, @eval_context).parse(true, nil, attrs, elems)
+          }
+
+        else
+          basepath = '/'
+          fname = path
+          require 'open-uri'
+          data = nil
+          open(uri) { |f| read = f.read }
+          ss = StringScanner.new(data)
+          V1Parser.new(ss, basepath, fname, @eval_context).parse(true, nil, attrs, elems)
+        end
+
+      rescue SystemCallError => e
+        cpe = ConfigParseError.new("include error #{uri}")
+        cpe.set_backtrace(e.backtrace)
+        raise cpe
+      end
+
+      # override
+      def error_sample
+        "#{@fname} #{super}"
+      end
+    end
+  end
+end