fluentd 0.10.26 → 0.10.27

data/ChangeLog

@@ -1,4 +1,10 @@
 
+Release 0.10.27 - 2012/10/11
+
+* added BufferChunk#unique_id which is an unique identifier of a buffered chunk
+* BufferedOutput: show 'temporarily failed' message
+
+
 Release 0.10.26 - 2012/09/26
 
 * added in_debug_agent plugin and fluent-debug command which enable you to

data/VERSION

@@ -1 +1 @@
-0.10.26
+0.10.27

data/lib/fluent/output.rb

@@ -333,7 +333,7 @@ class BufferedOutput < Output
       end
 
       if error_count < @retry_limit
-        $log.warn "failed to flush the buffer, retrying.", :error=>e.to_s, :instance=>object_id
+        $log.warn "temporarily failed to flush the buffer, next retry will be at #{Time.at(@next_retry_time)}.", :error=>e.to_s, :instance=>object_id
         $log.warn_backtrace e.backtrace
 
       elsif @secondary
@@ -343,7 +343,7 @@ class BufferedOutput < Output
           $log.warn_backtrace e.backtrace
           retry  # retry immediately
         elsif error_count <= @retry_limit + @secondary_limit
-          $log.warn "failed to flush the buffer, retrying secondary.", :error=>e.to_s, :instance=>object_id
+          $log.warn "failed to flush the buffer, next retry will be with secondary output at #{Time.at(@next_retry_time)}.", :error=>e.to_s, :instance=>object_id
           $log.warn_backtrace e.backtrace
         else
           $log.warn "failed to flush the buffer.", :error=>e.to_s, :instance=>object_id

data/lib/fluent/plugin.rb

@@ -120,7 +120,7 @@ class PluginClass
     }.compact
     unless files.empty?
       # prefer newer version
-      require files.sort.last
+      require File.expand_path(files.sort.last)
       return
     end
 

data/lib/fluent/plugin/buf_file.rb

@@ -19,14 +19,17 @@ module Fluent
 
 
 class FileBufferChunk < BufferChunk
-  def initialize(key, path, mode="a+")
+  def initialize(key, path, unique_id, mode="a+")
     super(key)
     @path = path
+    @unique_id = unique_id
     @file = File.open(@path, mode)
     @file.sync = true
     @size = @file.stat.size
   end
 
+  attr_reader :unique_id
+
   def <<(data)
     @file.write(data)
     @size += data.bytesize
@@ -103,8 +106,9 @@ class FileBuffer < BasicBuffer
 
   def new_chunk(key)
     encoded_key = encode_key(key)
-    path = make_path(encoded_key, "b")
-    FileBufferChunk.new(key, path)
+    path, tsuffix = make_path(encoded_key, "b")
+    unique_id = tsuffix_to_unique_id(tsuffix)
+    FileBufferChunk.new(key, path, unique_id)
   end
 
   def resume
@@ -116,28 +120,30 @@ class FileBuffer < BasicBuffer
       if m = PATH_MATCH.match(match)
         key = decode_key(m[1])
         bq = m[2]
-        tsuffix = m[3].to_i(16)
+        tsuffix = m[3]
+        timestamp = m[3].to_i(16)
+        unique_id = tsuffix_to_unique_id(tsuffix)
 
         if bq == 'b'
-          chunk = FileBufferChunk.new(key, path, "a+")
-          maps << [tsuffix, chunk]
+          chunk = FileBufferChunk.new(key, path, unique_id, "a+")
+          maps << [timestamp, chunk]
         elsif bq == 'q'
-          chunk = FileBufferChunk.new(key, path, "r")
-          queues << [tsuffix, chunk]
+          chunk = FileBufferChunk.new(key, path, unique_id, "r")
+          queues << [timestamp, chunk]
         end
       end
     }
 
     map = {}
-    maps.sort_by {|(tsuffix,chunk)|
-      tsuffix
-    }.each {|(tsuffix,chunk)|
+    maps.sort_by {|(timestamp,chunk)|
+      timestamp
+    }.each {|(timestamp,chunk)|
       map[chunk.key] = chunk
     }
 
-    queue = queues.sort_by {|(tsuffix,chunk)|
-      tsuffix
-    }.map {|(tsuffix,chunk)|
+    queue = queues.sort_by {|(timestamp,chunk)|
+      timestamp
+    }.map {|(timestamp,chunk)|
       chunk
     }
 
@@ -150,7 +156,8 @@ class FileBuffer < BasicBuffer
 
     m = PATH_MATCH.match(mp)
     encoded_key = m ? m[1] : ""
-    npath = make_path(encoded_key, "q")
+    tsuffix = m[3]
+    npath = "#{@buffer_path_prefix}#{encoded_key}.q#{tsuffix}#{@buffer_path_suffix}"
 
     chunk.mv(npath)
   end
@@ -166,8 +173,14 @@ class FileBuffer < BasicBuffer
 
   def make_path(encoded_key, bq)
     now = Time.now.utc
-    tsuffix = ((now.to_i*1000*1000+now.usec) << 12 | rand(0xfff)).to_s(16)
-    "#{@buffer_path_prefix}#{encoded_key}.#{bq}#{tsuffix}#{@buffer_path_suffix}"
+    timestamp = ((now.to_i*1000*1000+now.usec) << 12 | rand(0xfff))
+    tsuffix = timestamp.to_s(16)
+    path = "#{@buffer_path_prefix}#{encoded_key}.#{bq}#{tsuffix}#{@buffer_path_suffix}"
+    return path, tsuffix
+  end
+
+  def tsuffix_to_unique_id(tsuffix)
+    tsuffix.scan(/../).map {|x| x.to_i(16) }.pack('C*') * 2
   end
 end
 

data/lib/fluent/plugin/buf_memory.rb

@@ -22,9 +22,14 @@ class MemoryBufferChunk < BufferChunk
   def initialize(key, data='')
     @data = data
     @data.force_encoding('ASCII-8BIT')
+    now = Time.now.utc
+    u1 = ((now.to_i*1000*1000+now.usec) << 12 | rand(0xfff))
+    @unique_id = [u1 >> 32, u1 & u1 & 0xffffffff, rand(0xffffffff), rand(0xffffffff)].pack('NNNN')
     super(key)
   end
 
+  attr_reader :unique_id
+
   def <<(data)
     data.force_encoding('ASCII-8BIT')
     @data << data

data/lib/fluent/version.rb

@@ -1,5 +1,5 @@
 module Fluent
 
-VERSION = '0.10.26'
+VERSION = '0.10.27'
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.10.26
+  version: 0.10.27
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-26 00:00:00.000000000 Z
+date: 2012-10-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -193,8 +193,6 @@ files:
 - lib/fluent/mixin.rb
 - lib/fluent/output.rb
 - lib/fluent/parser.rb
-- lib/fluent/parser.rb.orig
-- lib/fluent/parser.rb.rej
 - lib/fluent/plugin.rb
 - lib/fluent/plugin/buf_file.rb
 - lib/fluent/plugin/buf_memory.rb

data/lib/fluent/parser.rb.orig

@@ -1,159 +0,0 @@
-#
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-module Fluent
-
-
-class TextParser
-  class RegexpParser
-    include Configurable
-
-    config_param :time_format, :string, :default => nil
-
-    def initialize(regexp, conf={})
-      super()
-      @regexp = regexp
-      unless conf.empty?
-        configure(conf)
-      end
-    end
-
-    def call(text)
-      m = @regexp.match(text)
-      unless m
-        $log.debug "pattern not match: #{text}"
-        # TODO?
-        return nil, nil
-      end
-
-      time = nil
-      record = {}
-
-      m.names.each {|name|
-        if value = m[name]
-          case name
-          when "time"
-            if @time_format
-              time = Time.strptime(value, @time_format).to_i
-            else
-              time = Time.parse(value).to_i
-            end
-          else
-            record[name] = value
-          end
-        end
-      }
-
-      time ||= Engine.now
-
-      return time, record
-    end
-  end
-
-  class JSONParser
-    include Configurable
-
-    config_param :time_key, :string, :default => 'time'
-    config_param :time_format, :string, :default => nil
-
-    def call(text)
-      record = Yajl.load(text)
-
-      if value = record.delete(@time_key)
-        if @time_format
-          time = Time.strptime(value, @time_format).to_i
-        else
-          time = value.to_i
-        end
-      else
-        time = Engine.now
-      end
-
-      return time, record
-    rescue Yajl::ParseError
-      # TODO?
-      return nil, nil
-    end
-  end
-
-  TEMPLATES = {
-    'apache' => RegexpParser.new(/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/, {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}),
-    'syslog' => RegexpParser.new(/^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?[^\:]*\: *(?<message>.*)$/, {'time_format'=>"%b %d %H:%M:%S"}),
-    'json' => JSONParser.new,
-  }
-
-  def self.register_template(name, regexp_or_proc, time_format=nil)
-    if regexp_or_proc.is_a?(Regexp)
-      regexp = regexp_or_proc
-      pr = RegexpParser.new(regexp, {'time_format'=>time_format})
-    else
-      pr = regexp_or_proc
-    end
-
-    TEMPLATES[name] = pr
-  end
-
-  def initialize
-    @parser = nil
-  end
-
-  def configure(conf, required=true)
-    format = conf['format']
-
-    if format == nil
-      if required
-        raise ConfigError, "'format' parameter is required"
-      else
-        return nil
-      end
-    end
-
-    if format[0] == ?/ && format[format.length-1] == ?/
-      # regexp
-      begin
-        regexp = Regexp.new(format[1..-2])
-        if regexp.named_captures.empty?
-          raise "No named captures"
-        end
-      rescue
-        raise ConfigError, "Invalid regexp '#{format[1..-2]}': #{$!}"
-      end
-
-      @parser = RegexpParser.new(regexp)
-
-    else
-      # built-in template
-      @parser = TEMPLATES[format]
-      unless @parser
-        raise ConfigError, "Unknown format template '#{format}'"
-      end
-    end
-
-    if @parser.respond_to?(:configure)
-      @parser.configure(conf)
-    end
-
-    return true
-  end
-
-  def parse(text)
-    return @parser.call(text)
-  end
-end
-
-
-end

data/lib/fluent/parser.rb.rej

@@ -1,91 +0,0 @@
-***************
-*** 90,107 ****
-      end
-    end
-  
-    TEMPLATES = {
--     'apache' => RegexpParser.new(/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/, {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}),
--     'syslog' => RegexpParser.new(/^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?[^\:]*\: *(?<message>.*)$/, {'time_format'=>"%b %d %H:%M:%S"}),
--     'json' => JSONParser.new,
-    }
-  
-    def self.register_template(name, regexp_or_proc, time_format=nil)
-      if regexp_or_proc.is_a?(Regexp)
--       pr = regexp_or_proc
-      else
-        regexp = regexp_or_proc
--       pr = RegexpParser.new(regexp, {'time_format'=>time_format})
-      end
-  
-      TEMPLATES[name] = pr
---- 90,159 ----
-      end
-    end
-  
-+   class ApacheParser
-+     include Configurable
-+ 
-+     REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
-+ 
-+     def call(text)
-+       m = REGEXP.match(text)
-+       unless m
-+         $log.debug "pattern not match: #{text}"
-+         return nil, nil
-+       end
-+ 
-+       host = m['host']
-+       host = (host == '-') ? nil : host
-+ 
-+       user = m['user']
-+       user = (user == '-') ? nil : user
-+ 
-+       time = m['time']
-+       time = Time.strptime(time, "%d/%b/%Y:%H:%M:%S %z").to_i
-+ 
-+       method = m['method']
-+       path = m['path']
-+ 
-+       code = m['code'].to_i
-+       code = nil if code == 0
-+ 
-+       size = m['size']
-+       size = (size == '-') ? nil : size.to_i
-+ 
-+       referer = m['referer']
-+       referer = (referer == '-') ? nil : referer
-+ 
-+       agent = m['agent']
-+       agent = (agent == '-') ? nil : agent
-+ 
-+       record = {
-+         "host" => host,
-+         "user" => user,
-+         "method" => method,
-+         "path" => path,
-+         "code" => code,
-+         "size" => size,
-+         "referer" => referer,
-+         "agent" => agent,
-+       }
-+ 
-+       return time, record
-+     end
-+   end
-+ 
-    TEMPLATES = {
-+     'apache' => Proc.new { RegexpParser.new(/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/, {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}) },
-+     'apache2' => Proc.new { ApacheParser.new },
-+     'syslog' => Proc.new { RegexpParser.new(/^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?[^\:]*\: *(?<message>.*)$/, {'time_format'=>"%b %d %H:%M:%S"}) },
-+     'json' => Proc.new { JSONParser.new },
-    }
-  
-    def self.register_template(name, regexp_or_proc, time_format=nil)
-      if regexp_or_proc.is_a?(Regexp)
-+       pr = Proc.new { regexp_or_proc }
-      else
-        regexp = regexp_or_proc
-+       pr = Proc.new { RegexpParser.new(regexp, {'time_format'=>time_format}) }
-      end
-  
-      TEMPLATES[name] = pr