fluentd 0.12.0.pre.1 → 0.12.0.pre.2

data/lib/fluent/config/element.rb

@@ -45,6 +45,7 @@ module Fluent
         "name:#{@name}, arg:#{@arg}, " + attrs + ", " + @elements.inspect
       end
 
+      # This method assumes _o_ is an Element object. Should return false for nil or other object
       def ==(o)
         self.name == o.name && self.arg == o.arg &&
           self.keys.size == o.keys.size &&
@@ -102,11 +103,7 @@ module Fluent
           out << "#{indent}<#{@name} #{@arg}>\n"
         end
         each_pair { |k, v|
-          if @v1_config
-            out << "#{nindent}#{k} #{Element.unescape_parameter(v)}\n"
-          else
-            out << "#{nindent}#{k} #{v}\n"
-          end
+          out << "#{nindent}#{k} #{v}\n"
         }
         @elements.each { |e|
           out << e.to_s(nest + 1)

data/lib/fluent/config/literal_parser.rb

@@ -51,7 +51,7 @@ module Fluent
       end
 
       def parse_literal(string_boundary_charset = LINE_END)
-        spacing
+        spacing_without_comment
 
         value = if skip(/\[/)
                   scan_json(true)
@@ -65,13 +65,15 @@ module Fluent
 
       def scan_string(string_boundary_charset = LINE_END)
         if skip(/\"/)
-          return scan_quoted_string
+          return scan_double_quoted_string
+        elsif skip(/\'/)
+          return scan_single_quoted_string
         else
           return scan_nonquoted_string(string_boundary_charset)
         end
       end
 
-      def scan_quoted_string
+      def scan_double_quoted_string
         string = []
         while true
           if skip(/\"/)
@@ -84,7 +86,24 @@ module Fluent
           elsif s = scan(/./)
             string << s
           else
-            parse_error! "unexpected end of file in a quoted string"
+            parse_error! "unexpected end of file in a double quoted string"
+          end
+        end
+      end
+
+      def scan_single_quoted_string
+        string = []
+        while true
+          if skip(/\'/)
+            return string.join
+          elsif s = scan(/\\'/)
+            string << "'"
+          elsif s = scan(/\\\\/)
+            string << "\\"
+          elsif s = scan(/./)
+            string << s
+          else
+            parse_error! "unexpected end of file in a signle quoted string"
           end
         end
       end
@@ -94,8 +113,8 @@ module Fluent
 
         string = []
         while true
-          if s = scan(/\\./)
-            string << eval_escape_char(s[1,1])
+          if s = scan(/\#/)
+            string << '#'
           elsif s = scan(charset)
             string << s
           else
@@ -214,7 +233,7 @@ module Fluent
         end
 
         unless result
-          parse_error! "got incomplete #{is_array ? 'array' : 'hash'} configuration"
+          parse_error! "got incomplete JSON #{is_array ? 'array' : 'hash'} configuration"
         end
 
         JSON.dump(result)

data/lib/fluent/config/section.rb

@@ -30,6 +30,8 @@ module Fluent
         @params = params
       end
 
+      alias :object_id :__id__
+
       def inspect
         "<Fluent::Config::Section #{@params.to_json}>"
       end

data/lib/fluent/config/v1_parser.rb

@@ -50,6 +50,8 @@ module Fluent
         return root
       end
 
+      ELEM_SYMBOLS = ['match', 'source', 'filter', 'system']
+
       def parse_element(root_element, elem_name, attrs = {}, elems = [])
         while true
           spacing
@@ -100,7 +102,7 @@ module Fluent
 
           else
             k = scan_string(SPACING)
-            spacing
+            spacing_without_comment
             if prev_match.include?("\n") # support 'tag_mapped' like "without value" configuration
               attrs[k] = ""
             else
@@ -114,7 +116,12 @@ module Fluent
                 attrs[k] = v
               else
                 if k.start_with?('@')
-                  parse_error! "'@' is reserved prefix. Don't use '@' in parameter name"
+                  if root_element || ELEM_SYMBOLS.include?(elem_name)
+                    parse_error! "'@' is the system reserved prefix. Don't use '@' prefix parameter in the configuration: #{k}"
+                  else
+                    # TODO: This is for backward compatibility. It will throw an error in the future.
+                    $log.warn "'@' is the system reserved prefix. It works in the nested configuration for now but it will be rejected: #{k}"
+                  end
                 end
 
                 v = parse_literal

data/lib/fluent/formatter.rb

@@ -27,6 +27,7 @@ module Fluent
           config_param :include_tag_key, :bool, :default => false
           config_param :tag_key, :string, :default => 'tag'
           config_param :localtime, :bool, :default => true
+          config_param :timezone, :string, :default => nil
         }
       end
 
@@ -36,7 +37,7 @@ module Fluent
         if conf['utc']
           @localtime = false
         end
-        @timef = TimeFormatter.new(@time_format, @localtime)
+        @timef = TimeFormatter.new(@time_format, @localtime, @timezone)
       end
 
       def filter_record(tag, time, record)

data/lib/fluent/mixin.rb

@@ -16,12 +16,21 @@
 
 module Fluent
   class TimeFormatter
-    def initialize(format, localtime)
+    require 'fluent/timezone'
+
+    def initialize(format, localtime, timezone = nil)
       @tc1 = 0
       @tc1_str = nil
       @tc2 = 0
       @tc2_str = nil
 
+      if formatter = Fluent::Timezone.formatter(timezone, format)
+        define_singleton_method(:format_nocache) {|time|
+          formatter.call(time)
+        }
+        return
+      end
+
       if format
         if localtime
           define_singleton_method(:format_nocache) {|time|
@@ -112,15 +121,16 @@ module Fluent
   end
 
   module SetTimeKeyMixin
+    require 'fluent/timezone'
     include RecordFilterMixin
 
-    attr_accessor :include_time_key, :time_key, :localtime
+    attr_accessor :include_time_key, :time_key, :localtime, :timezone
 
     def configure(conf)
-      super
-
       @include_time_key = false
 
+      super
+
       if s = conf['include_time_key']
         include_time_key = Config.bool_value(s)
         raise ConfigError, "Invalid boolean expression '#{s}' for include_time_key parameter" if include_time_key.nil?
@@ -138,7 +148,12 @@ module Fluent
           @localtime = false
         end
 
-        @timef = TimeFormatter.new(@time_format, @localtime)
+        if conf['timezone']
+          @timezone = conf['timezone']
+          Fluent::Timezone.validate!(@timezone)
+        end
+
+        @timef = TimeFormatter.new(@time_format, @localtime, @timezone)
       end
     end
 
@@ -155,10 +170,10 @@ module Fluent
     attr_accessor :include_tag_key, :tag_key
 
     def configure(conf)
-      super
-
       @include_tag_key = false
 
+      super
+
       if s = conf['include_tag_key']
         include_tag_key = Config.bool_value(s)
         raise ConfigError, "Invalid boolean expression '#{s}' for include_tag_key parameter" if include_tag_key.nil?

data/lib/fluent/output.rb

@@ -322,7 +322,7 @@ module Fluent
           @num_errors = 0
           # Note: don't notify to other threads to prevent
           #       burst to recovered server
-          $log.warn "retry succeeded.", :instance=>object_id
+          $log.warn "retry succeeded.", :plugin_id=>plugin_id
         end
 
         if has_next
@@ -347,20 +347,20 @@ module Fluent
         end
 
         if @disable_retry_limit || error_count < @retry_limit
-          $log.warn "temporarily failed to flush the buffer.", :next_retry=>Time.at(@next_retry_time), :error_class=>e.class.to_s, :error=>e.to_s, :instance=>object_id
+          $log.warn "temporarily failed to flush the buffer.", :next_retry=>Time.at(@next_retry_time), :error_class=>e.class.to_s, :error=>e.to_s, :plugin_id=>plugin_id
           $log.warn_backtrace e.backtrace
 
         elsif @secondary
           if error_count == @retry_limit
-            $log.warn "failed to flush the buffer.", :error_class=>e.class.to_s, :error=>e.to_s, :instance=>object_id
+            $log.warn "failed to flush the buffer.", :error_class=>e.class.to_s, :error=>e.to_s, :plugin_id=>plugin_id
             $log.warn "retry count exceededs limit. falling back to secondary output."
             $log.warn_backtrace e.backtrace
             retry  # retry immediately
           elsif error_count <= @retry_limit + @secondary_limit
-            $log.warn "failed to flush the buffer, next retry will be with secondary output.", :next_retry=>Time.at(@next_retry_time), :error_class=>e.class.to_s, :error=>e.to_s, :instance=>object_id
+            $log.warn "failed to flush the buffer, next retry will be with secondary output.", :next_retry=>Time.at(@next_retry_time), :error_class=>e.class.to_s, :error=>e.to_s, :plugin_id=>plugin_id
             $log.warn_backtrace e.backtrace
           else
-            $log.warn "failed to flush the buffer.", :error_class=>e.class, :error=>e.to_s, :instance=>object_id
+            $log.warn "failed to flush the buffer.", :error_class=>e.class, :error=>e.to_s, :plugin_id=>plugin_id
             $log.warn "secondary retry count exceededs limit."
             $log.warn_backtrace e.backtrace
             write_abort
@@ -368,7 +368,7 @@ module Fluent
           end
 
         else
-          $log.warn "failed to flush the buffer.", :error_class=>e.class.to_s, :error=>e.to_s, :instance=>object_id
+          $log.warn "failed to flush the buffer.", :error_class=>e.class.to_s, :error=>e.to_s, :plugin_id=>plugin_id
           $log.warn "retry count exceededs limit."
           $log.warn_backtrace e.backtrace
           write_abort
@@ -459,6 +459,8 @@ module Fluent
 
 
   class TimeSlicedOutput < BufferedOutput
+    require 'fluent/timezone'
+
     def initialize
       super
       @localtime = true
@@ -467,6 +469,7 @@ module Fluent
 
     config_param :time_slice_format, :string, :default => '%Y%m%d'
     config_param :time_slice_wait, :time, :default => 10*60
+    config_param :timezone, :string, :default => nil
     config_set_default :buffer_type, 'file'  # overwrite default buffer_type
     config_set_default :buffer_chunk_limit, 256*1024*1024  # overwrite default buffer_chunk_limit
     config_set_default :flush_interval, nil
@@ -476,14 +479,20 @@ module Fluent
     def configure(conf)
       super
 
-      # TODO timezone
       if conf['utc']
         @localtime = false
       elsif conf['localtime']
         @localtime = true
       end
 
-      if @localtime
+      if conf['timezone']
+        @timezone = conf['timezone']
+        Fluent::Timezone.validate!(@timezone)
+      end
+
+      if @timezone
+        @time_slicer = Timezone.formatter(@timezone, @time_slice_format)
+      elsif @localtime
         @time_slicer = Proc.new {|time|
           Time.at(time).strftime(@time_slice_format)
         }

data/lib/fluent/parser.rb

@@ -18,6 +18,9 @@ module Fluent
   require 'fluent/registry'
 
   class TextParser
+    class ParserError < StandardError
+    end
+
     class TimeParser
       def initialize(time_format)
         @cache1_key = nil
@@ -34,7 +37,7 @@ module Fluent
 
       def parse(value)
         unless value.is_a?(String)
-          raise ArgumentError, "Value must be string: #{value}"
+          raise ParserError, "value must be string: #{value}"
         end
 
         if @cache1_key == value
@@ -42,7 +45,11 @@ module Fluent
         elsif @cache2_key == value
           return @cache2_time
         else
-          time = @parser.call(value).to_i
+          begin
+            time = @parser.call(value).to_i
+          rescue => e
+            raise ParserError, "invalid time format: value = #{value}, error_class = #{e.class.name}, error = #{e.message}"
+          end
           @cache1_key = @cache2_key
           @cache1_time = @cache2_time
           @cache2_key = value
@@ -224,7 +231,11 @@ module Fluent
           if @time_format
             time = @mutex.synchronize { @time_parser.parse(value) }
           else
-            time = value.to_i
+            begin
+              time = value.to_i
+            rescue => e
+              raise ParserError, "invalid time value: value = #{value}, error_class = #{e.class.name}, error = #{e.message}"
+            end
           end
         else
           if @estimate_current_event

data/lib/fluent/plugin/buf_file.rb

@@ -26,7 +26,7 @@ module Fluent
       FileUtils.ln_sf(@path, symlink_path) if symlink_path
     end
 
-    attr_reader :unique_id
+    attr_reader :unique_id, :path
 
     def <<(data)
       @file.write(data)
@@ -64,8 +64,6 @@ module Fluent
       yield @file
     end
 
-    attr_reader :path
-
     def mv(path)
       File.rename(@path, path)
       @path = path
@@ -80,10 +78,15 @@ module Fluent
     def initialize
       require 'uri'
       super
+
+      @uri_parser = URI::Parser.new
     end
 
     config_param :buffer_path, :string
 
+    # 'symlink_path' is currently only for out_file.
+    # That is the reason why this is not config_param, but attr_accessor.
+    # See: https://github.com/fluent/fluentd/pull/181
     attr_accessor :symlink_path
 
     def configure(conf)
@@ -96,10 +99,10 @@ module Fluent
       end
 
       if pos = @buffer_path.index('*')
-        @buffer_path_prefix = @buffer_path[0,pos]
-        @buffer_path_suffix = @buffer_path[pos+1..-1]
+        @buffer_path_prefix = @buffer_path[0, pos]
+        @buffer_path_suffix = @buffer_path[(pos + 1)..-1]
       else
-        @buffer_path_prefix = @buffer_path+"."
+        @buffer_path_prefix = @buffer_path + "."
         @buffer_path_suffix = ".log"
       end
 
@@ -111,11 +114,13 @@ module Fluent
     end
 
     def start
-      FileUtils.mkdir_p File.dirname(@buffer_path_prefix+"path")
+      FileUtils.mkdir_p File.dirname(@buffer_path_prefix + "path")
       super
     end
 
-    PATH_MATCH = /^(.*)[\._](b|q)([0-9a-fA-F]{1,32})$/
+    # Dots are separator for many cases:
+    #   we should have to escape dots in keys...
+    PATH_MATCH = /^([-_.%0-9a-zA-Z]*)\.(b|q)([0-9a-fA-F]{1,32})$/
 
     def new_chunk(key)
       encoded_key = encode_key(key)
@@ -129,8 +134,8 @@ module Fluent
       queues = []
 
       Dir.glob("#{@buffer_path_prefix}*#{@buffer_path_suffix}") {|path|
-        match = path[@buffer_path_prefix.length..-(@buffer_path_suffix.length+1)]
-        if m = PATH_MATCH.match(match)
+        identifier_part = chunk_identifier_in_path(path)
+        if m = PATH_MATCH.match(identifier_part)
           key = decode_key(m[1])
           bq = m[2]
           tsuffix = m[3]
@@ -163,11 +168,18 @@ module Fluent
       return queue, map
     end
 
+    def chunk_identifier_in_path(path)
+      pos_after_prefix = @buffer_path_prefix.length
+      pos_before_suffix = @buffer_path_suffix.length + 1 # from tail of path
+
+      path.slice(pos_after_prefix..-pos_before_suffix)
+    end
+
     def enqueue(chunk)
       path = chunk.path
-      mp = path[@buffer_path_prefix.length..-(@buffer_path_suffix.length+1)]
+      identifier_part = chunk_identifier_in_path(path)
 
-      m = PATH_MATCH.match(mp)
+      m = PATH_MATCH.match(identifier_part)
       encoded_key = m ? m[1] : ""
       tsuffix = m[3]
       npath = "#{@buffer_path_prefix}#{encoded_key}.q#{tsuffix}#{@buffer_path_suffix}"
@@ -189,23 +201,26 @@ module Fluent
 
     protected
 
+    # Dots are separator for many cases:
+    #   we should have to escape dots in keys...
     def encode_key(key)
-      URI.escape(key, /[^-_.a-zA-Z0-9]/n)
+      @uri_parser.escape(key, /[^-_.a-zA-Z0-9]/n) # //n switch means explicit 'ASCII-8BIT' pattern
     end
 
     def decode_key(encoded_key)
-      URI.unescape(encoded_key)
+      @uri_parser.unescape(encoded_key)
     end
 
     def make_path(encoded_key, bq)
       now = Time.now.utc
-      timestamp = ((now.to_i*1000*1000+now.usec) << 12 | rand(0xfff))
+      timestamp = ((now.to_i * 1000 * 1000 + now.usec) << 12 | rand(0xfff))
       tsuffix = timestamp.to_s(16)
       path = "#{@buffer_path_prefix}#{encoded_key}.#{bq}#{tsuffix}#{@buffer_path_suffix}"
       return path, tsuffix
     end
 
     def tsuffix_to_unique_id(tsuffix)
+      # why *2 ? frsyuki said that I forgot why completely.
       tsuffix.scan(/../).map {|x| x.to_i(16) }.pack('C*') * 2
     end
   end