fluentd 0.10.45 → 0.10.46

data/lib/fluent/config/basic_parser.rb

@@ -0,0 +1,108 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+  module Config
+
+    require 'stringio'
+    require 'fluent/config/error'
+
+    class BasicParser
+      def initialize(strscan)
+        @ss = strscan
+      end
+
+      LINE_END = /(?:[ \t]*(?:\#.*)?(?:\z|[\r\n]))+/
+      SPACING = /(?:[ \t\r\n]|\z|\#.*?(?:\z|[\r\n]))+/
+
+      module ClassMethods
+        def symbol(string)
+          /#{Regexp.escape(string)}/
+        end
+
+        def def_symbol(method_name, string)
+          pattern = symbol(string)
+          define_method(method_name) do
+            skip(pattern) && string
+          end
+        end
+
+        def def_literal(method_name, string)
+          pattern = /#{string}#{LINE_END}/
+          define_method(method_name) do
+            skip(pattern) && string
+          end
+        end
+      end
+
+      extend ClassMethods
+
+      def skip(pattern)
+        @ss.skip(pattern)
+      end
+
+      def scan(pattern)
+        @ss.scan(pattern)
+      end
+
+      def getch
+        @ss.getch
+      end
+
+      def eof?
+        @ss.eos?
+      end
+
+      def line_end
+        skip(LINE_END)
+      end
+
+      def spacing
+        skip(SPACING)
+      end
+
+      def parse_error!(message)
+        raise ConfigParseError, "#{message} at #{error_sample}"
+      end
+
+      def error_sample
+        pos = @ss.pos
+
+        lines = @ss.string.lines.to_a
+        lines.each_with_index { |line, ln|
+          if line.size >= pos
+            msgs = ["line #{ln + 1},#{pos}\n"]
+
+            if ln > 0
+              last_line = lines[ln - 1]
+              msgs << "%3s: %s" % [ln, last_line]
+            end
+
+            msgs << "%3s: %s" % [ln + 1, line]
+            msgs << "\n     #{'-' * pos}^\n"
+
+            if next_line = lines[ln + 1]
+              msgs << "%3s: %s" % [ln + 2, next_line]
+            end
+
+            return msgs.join
+          end
+          pos -= line.size
+          last_line = line
+        }
+      end
+    end
+  end
+end

data/lib/fluent/config/configure_proxy.rb

@@ -0,0 +1,145 @@
+module Fluent
+  module Config
+    class ConfigureProxy
+      attr_accessor :name, :param_name, :required, :multi, :argument, :params, :defaults, :sections
+      # config_param :desc, :string, :default => '....'
+      # config_set_default :buffer_type, :memory
+      #
+      # config_section :default, required: true, multi: false do
+      #   config_argument :arg, :string
+      #   config_param :required, :bool, default: false
+      #   config_param :name, :string
+      #   config_param :power, :integer
+      # end
+      #
+      # config_section :child, param_name: 'children', required: false, multi: true do
+      #   config_param :name, :string
+      #   config_param :power, :integer, default: nil
+      #   config_section :item do
+      #     config_param :name
+      #   end
+      # end
+
+      def initialize(name, opts = {})
+        @name = name.to_sym
+
+        @param_name = (opts[:param_name] || @name).to_sym
+        @required = opts[:required]
+        @multi = opts[:multi]
+
+        @argument = nil # nil: ignore argument
+        @params = {}
+        @defaults = {}
+        @sections = {}
+      end
+
+      def required?
+        @required.nil? ? false : @required
+      end
+
+      def multi?
+        @multi.nil? ? true : @multi
+      end
+
+      def merge(other) # self is base class, other is subclass
+        options = {
+          param_name: other.param_name,
+          required: (other.required.nil? ? self.required : other.required),
+          multi: (other.multi.nil? ? self.multi : other.multi)
+        }
+        merged = self.class.new(other.name, options)
+
+        merged.argument = other.argument || self.argument
+        merged.params = self.params.merge(other.params)
+        merged.defaults = self.defaults.merge(other.defaults)
+        merged.sections = self.sections.merge(other.sections)
+
+        merged
+      end
+
+      def parameter_configuration(name, *args, &block)
+        name = name.to_sym
+
+        opts = {}
+        args.each { |a|
+          if a.is_a?(Symbol)
+            opts[:type] = a
+          elsif a.is_a?(Hash)
+            opts.merge!(a)
+          else
+            raise ArgumentError, "wrong number of arguments (#{1 + args.length} for #{block ? 2 : 3})"
+          end
+        }
+
+        type = opts[:type]
+        if block && type
+          raise ArgumentError, "both of block and type cannot be specified"
+        end
+
+        begin
+          type = :string if type.nil?
+          block ||= Configurable.lookup_type(type)
+        rescue ConfigError
+          # override error message
+          raise ArgumentError, "unknown config_argument type `#{type}'"
+        end
+
+        if opts.has_key?(:default)
+          config_set_default(name, opts[:default])
+        end
+
+        [name, block, opts]
+      end
+
+      def config_argument(name, *args, &block)
+        if @argument
+          raise ArgumentError, "config_argument called twice"
+        end
+        name, block, opts = parameter_configuration(name, *args, &block)
+
+        @argument = [name, block, opts]
+        name
+      end
+
+      def config_param(name, *args, &block)
+        name, block, opts = parameter_configuration(name, *args, &block)
+
+        @sections.delete(name)
+        @params[name] = [block, opts]
+        name
+      end
+
+      def config_set_default(name, defval)
+        name = name.to_sym
+
+        if @defaults.has_key?(name)
+          raise ArgumentError, "default value specified twice for #{name}"
+        end
+
+        @defaults[name] = defval
+        nil
+      end
+
+      def config_section(name, *args, &block)
+        unless block_given?
+          raise ArgumentError, "config_section requires block parameter"
+        end
+        name = name.to_sym
+
+        opts = {}
+        unless args.empty? || args.size == 1 && args.first.is_a?(Hash)
+          raise ArgumentError, "unknown config_section arguments: #{args.inspect}"
+        end
+
+        sub_proxy = ConfigureProxy.new(name, *args)
+        sub_proxy.instance_exec(&block)
+
+        @params.delete(name)
+        @sections[name] = sub_proxy
+
+        name
+      end
+    end
+  end
+end
+

data/lib/fluent/{config_dsl.rb → config/dsl.rb}

@@ -67,7 +67,11 @@ module Fluent
             proxy.element.instance_exec(&block)
             @elements.push(proxy.to_config_element)
           else
-            @attrs[name.to_s] = value.to_s
+            @attrs[name.to_s] = if value.is_a?(Array) || value.is_a?(Hash)
+                                  JSON.dump(value)
+                                else
+                                  value.to_s
+                                end
           end
 
           self

data/lib/fluent/config/element.rb

@@ -0,0 +1,82 @@
+module Fluent
+  require 'fluent/config/error'
+
+  module Config
+    class Element < Hash
+      def initialize(name, arg, attrs, elements, unused = nil)
+        @name = name
+        @arg = arg
+        @elements = elements
+        super()
+        attrs.each { |k, v|
+          self[k] = v
+        }
+        @unused = unused || attrs.keys
+      end
+
+      attr_accessor :name, :arg, :elements, :unused
+
+      def add_element(name, arg='')
+        e = Element.new(name, arg, {}, [])
+        @elements << e
+        e
+      end
+
+      def +(o)
+        Element.new(@name.dup, @arg.dup, o.merge(self), @elements + o.elements, (@unused + o.unused).uniq)
+      end
+
+      def each_element(*names, &block)
+        if names.empty?
+          @elements.each(&block)
+        else
+          @elements.each { |e|
+            if names.include?(e.name)
+              block.yield(e)
+            end
+          }
+        end
+      end
+
+      def has_key?(key)
+        @unused.delete(key)
+        super
+      end
+
+      def [](key)
+        @unused.delete(key)
+        super
+      end
+
+      def check_not_fetched(&block)
+        each_key { |key|
+          if @unused.include?(key)
+            block.call(key, self)
+          end
+        }
+        @elements.each { |e|
+          e.check_not_fetched(&block)
+        }
+      end
+
+      def to_s(nest = 0)
+        indent = "  " * nest
+        nindent = "  " * (nest + 1)
+        out = ""
+        if @arg.empty?
+          out << "#{indent}<#{@name}>\n"
+        else
+          out << "#{indent}<#{@name} #{@arg}>\n"
+        end
+        each_pair { |k, v|
+          out << "#{nindent}#{k} #{v}\n"
+        }
+        @elements.each { |e|
+          out << e.to_s(nest + 1)
+        }
+        out << "#{indent}</#{@name}>\n"
+        out
+      end
+    end
+  end
+end

data/lib/fluent/config/error.rb

@@ -0,0 +1,7 @@
+module Fluent
+  class ConfigError < StandardError
+  end
+
+  class ConfigParseError < ConfigError
+  end
+end

data/lib/fluent/config/literal_parser.rb

@@ -0,0 +1,158 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+  module Config
+
+    require 'yajl'
+    require 'fluent/config/basic_parser'
+    require 'irb/ruby-lex'  # RubyLex
+
+    class LiteralParser < BasicParser
+      def initialize(strscan, eval_context)
+        super(strscan)
+        @eval_context = eval_context
+      end
+
+      def parse_literal(string_boundary_charset = LINE_END)
+        spacing
+
+        value = if skip(/\[/)
+                  scan_json(true)
+                elsif skip(/\{/)
+                  scan_json(false)
+                else
+                  scan_string(string_boundary_charset)
+                end
+        value
+      end
+
+      def scan_string(string_boundary_charset = LINE_END)
+        if skip(/\"/)
+          return scan_quoted_string
+        else
+          return scan_nonquoted_string(string_boundary_charset)
+        end
+      end
+
+      def scan_quoted_string
+        string = []
+        while true
+          if skip(/\"/)
+            return string.join
+          elsif s = scan(/\\./)
+            string << eval_escape_char(s[1,1])
+          elsif skip(/\#\{/)
+            string << eval_embedded_code(scan_embedded_code)
+            skip(/\}/)
+          elsif s = scan(/./)
+            string << s
+          else
+            parse_error! "unexpected end of file in a quoted string"
+          end
+        end
+      end
+
+      def scan_nonquoted_string(boundary_charset = LINE_END)
+        charset = /(?!#{boundary_charset})./
+
+        string = []
+        while true
+          if s = scan(/\\./)
+            string << eval_escape_char(s[1,1])
+          elsif s = scan(charset)
+            string << s
+          else
+            break
+          end
+        end
+
+        if string.empty?
+          return nil
+        end
+
+        string.join
+      end
+
+      def scan_embedded_code
+        rlex = RubyLex.new
+        src = '"#{'+@ss.rest+"\n=end\n}"
+
+        input = StringIO.new(src)
+        input.define_singleton_method(:encoding) { external_encoding }
+        rlex.set_input(input)
+
+        tk = rlex.token
+        code = src[3,tk.seek-3]
+
+        if @ss.rest.length < code.length
+          @ss.pos += @ss.rest.length
+          parse_error! "expected end of embedded code but $end"
+        end
+
+        @ss.pos += code.length
+
+        code
+      end
+
+      def eval_embedded_code(code)
+        if @eval_context.nil?
+          parse_error! "embedded code is not allowed in this file"
+        end
+        @eval_context.instance_eval(code)
+      end
+
+      def eval_escape_char(c)
+        case c
+        when '"'
+          '"'
+        when "'"
+          "'"
+        when "r"
+          "\r"
+        when "n"
+          "\n"
+        when "t"
+          "\t"
+        when "f"
+          "\f"
+        when "b"
+          "\b"
+        when /[a-zA-Z0-9]/
+          parse_error! "unexpected back-slash escape character '#{c}'"
+        else  # symbols
+          c
+        end
+      end
+
+      def scan_json(is_array)
+        result = nil
+
+        y = Yajl::Parser.new(:allow_comments => true)
+        y.on_parse_complete = Proc.new { |obj| result = obj }
+        y << (is_array ? '[' : '{')
+        until result
+          ch = getch
+          parse_error! "got incomplete #{is_array ? 'array' : 'hash'} configuration" if ch.nil?
+          y << ch
+        end
+
+        Yajl.dump(result) # Convert json to string for config_param
+      rescue Yajl::ParseError => e
+        parse_error! "unexpected #{is_array ? 'array' : 'hash'} parse error"
+      end
+    end
+  end
+end