fluentd 0.10.20 → 0.10.21

data/ChangeLog

@@ -1,4 +1,14 @@
 
+Release 0.10.21 - 2012/05/01
+
+* in_tail and TextParser support 'format json'
+* in_tail and TextParser support 'time_key' and 'time_format' parameters for 'format json'
+* in_tail: improved the file reader not to consume too much memory if the file is huge
+* out_stdout: uses logger instead of STDOUT to show records even if Fluentd is daemonized
+* Fixed the type of 'retry_wait' parameter of buffered output plugins from 'float' to 'time'
+* Fixed owner of log files for log rotation to work properly with --user and --group options
+
+
 Release 0.10.20 - 2012/04/27
 
 * Fixed Config#to_s

data/VERSION

@@ -1 +1 @@
-0.10.20
+0.10.21

data/lib/fluent/log.rb

@@ -227,6 +227,14 @@ class Log
     nil
   end
 
+  def write(data)
+    @out.write(data)
+  end
+
+  def flush
+    @out.flush
+  end
+
   private
   def event(level, args)
     time = Time.now

data/lib/fluent/output.rb

@@ -174,7 +174,7 @@ class BufferedOutput < Output
   config_param :buffer_type, :string, :default => 'memory'
   config_param :flush_interval, :time, :default => 60
   config_param :retry_limit, :integer, :default => 17
-  config_param :retry_wait, :float, :default => 1.0
+  config_param :retry_wait, :time, :default => 1.0
   config_param :num_threads, :integer, :default => 1
 
   def configure(conf)

data/lib/fluent/parser.rb

@@ -19,95 +19,136 @@ module Fluent
 
 
 class TextParser
-  TEMPLATES = {
-    'apache' => [/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/, "%d/%b/%Y:%H:%M:%S %z"],
-    'syslog' => [/^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?[^\:]*\: *(?<message>.*)$/, "%b %d %H:%M:%S"],
-  }
+  class RegexpParser
+    include Configurable
 
-  def self.register_template(name, regexp, time_format=nil)
-    TEMPLATES[name] = [regexp, time_format]
-  end
+    config_param :time_format, :string
 
-  def self.get_template(name)
-    return *TEMPLATES[name]
-  end
+    def initialize(regexp, conf={})
+      super()
+      @regexp = regexp
+      unless conf.empty?
+        configure(conf)
+      end
+    end
 
-  def initialize
-    require 'time'  # Time.strptime, Time.parse
-    @regexp = nil
-    @time_format = nil
-  end
+    def call(text)
+      m = @regexp.match(text)
+      unless m
+        $log.debug "pattern not match: #{text}"
+        # TODO?
+        return nil, nil
+      end
+
+      time = nil
+      record = {}
+
+      m.names.each {|name|
+        if value = m[name]
+          case name
+          when "time"
+            if @time_format
+              time = Time.strptime(value, @time_format).to_i
+            else
+              time = Time.parse(value).to_i
+            end
+          else
+            record[name] = value
+          end
+        end
+      }
 
-  attr_accessor :regexp, :time_format
+      time ||= Engine.now
 
-  def use_template(name)
-    @regexp, @time_format = TextParser.get_template(name)
-    unless @regexp
-      raise ConfigError, "Unknown format template '#{name}'"
+      return time, record
     end
   end
 
-  def configure(conf, required=true)
-    if format = conf['format']
-      if format[0] == ?/ && format[format.length-1] == ?/
-        # regexp
-        begin
-          @regexp = Regexp.new(format[1..-2])
-          if @regexp.named_captures.empty?
-            raise "No named captures"
-          end
-        rescue
-          raise ConfigError, "Invalid regexp '#{format[1..-2]}': #{$!}"
-        end
+  class JSONParser
+    include Configurable
+
+    config_param :time_key, :string, :default => 'time'
+    config_param :time_format, :string, :default => nil
 
+    def call(text)
+      record = Yajl.load(text)
+
+      if value = record.delete(@time_key)
+        if @time_format
+          time = Time.strptime(value, @time_format).to_i
+        else
+          time = value.to_i
+        end
       else
-        # template
-        use_template(format)
+        time = Engine.now
       end
-    else
-      return nil if !required
-      raise ConfigError, "'format' parameter is required"
+
+      return time, record
     end
+  end
 
-    if time_format = conf['time_format']
-      unless @regexp.names.include?('time')
-        raise ConfigError, "'time_format' parameter is invalid when format doesn't have 'time' capture"
-      end
-      @time_format = time_format
+  TEMPLATES = {
+    'apache' => RegexpParser.new(/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/, {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}),
+    'syslog' => RegexpParser.new(/^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?[^\:]*\: *(?<message>.*)$/, {'time_format'=>"%b %d %H:%M:%S"}),
+    'json' => JSONParser.new,
+  }
+
+  def self.register_template(name, regexp_or_proc, time_format=nil)
+    if regexp_or_proc.is_a?(Regexp)
+      pr = regexp_or_proc
+    else
+      regexp = regexp_or_proc
+      pr = RegexpParser.new(regexp, {'time_format'=>time_format})
     end
 
-    return true
+    TEMPLATES[name] = pr
   end
 
-  def parse(text)
-    m = @regexp.match(text)
-    unless m
-      $log.debug "pattern not match: #{text}"
-      # TODO?
-      return nil, nil
-    end
+  def initialize
+    @parser = nil
+  end
 
-    time = nil
-    record = {}
+  def configure(conf, required=true)
+    format = conf['format']
 
-    m.names.each {|name|
-      if value = m[name]
-        case name
-        when "time"
-          if @time_format
-            time = Time.strptime(value, @time_format).to_i
-          else
-            time = Time.parse(value).to_i
-          end
-        else
-          record[name] = value
+    if format == nil
+      if required
+        raise ConfigError, "'format' parameter is required"
+      else
+        return nil
+      end
+    end
+
+    if format[0] == ?/ && format[format.length-1] == ?/
+      # regexp
+      begin
+        regexp = Regexp.new(format[1..-2])
+        if regexp.named_captures.empty?
+          raise "No named captures"
         end
+      rescue
+        raise ConfigError, "Invalid regexp '#{format[1..-2]}': #{$!}"
+      end
+
+      @parser = RegexpParser.new(regexp)
+
+    else
+      # built-in template
+      @parser = TEMPLATES[format]
+      unless @parser
+        raise ConfigError, "Unknown format template '#{format}'"
       end
-    }
+    end
+
+    if @parser.respond_to?(:configure)
+      @parser.configure(conf)
+    end
 
-    time ||= Engine.now
+    return true
+  end
 
-    return time, record
+  def parse(text)
+    return @parser.call(text)
   end
 end
 

data/lib/fluent/plugin/in_tail.rb

@@ -255,6 +255,8 @@ class TailInput < Input
       end
     end
 
+    MAX_LINES_AT_ONCE = 1000
+
     class IOHandler
       def initialize(io, pe, &receive_lines)
         $log.info "following tail of #{io.path}"
@@ -268,27 +270,36 @@ class TailInput < Input
       attr_reader :io
 
       def on_notify
-        lines = []
+        begin
+          lines = []
+          read_more = false
 
-        while true
           begin
-            if @buffer.empty?
-              @io.read_nonblock(2048, @buffer)
-            else
-              @buffer << @io.read_nonblock(2048, @iobuf)
-            end
-            while line = @buffer.slice!(/.*?\n/m)
-              lines << line
+            while true
+              if @buffer.empty?
+                @io.read_nonblock(2048, @buffer)
+              else
+                @buffer << @io.read_nonblock(2048, @iobuf)
+              end
+              while line = @buffer.slice!(/.*?\n/m)
+                lines << line
+              end
+              if lines.size >= MAX_LINES_AT_ONCE
+                # not to use too much memory in case the file is very large
+                read_more = true
+                break
+              end
             end
           rescue EOFError
-            break
           end
-        end
 
-        unless lines.empty?
-          @receive_lines.call(lines)
-          @pe.update_pos(@io.pos - @buffer.bytesize)
-        end
+          unless lines.empty?
+            @receive_lines.call(lines)
+            @pe.update_pos(@io.pos - @buffer.bytesize)
+          end
+
+        end while read_more
+
       rescue
         $log.error $!.to_s
         $log.error_backtrace

data/lib/fluent/plugin/out_stdout.rb

@@ -21,13 +21,11 @@ module Fluent
 class StdoutOutput < Output
   Plugin.register_output('stdout', self)
 
-  config_param :autoflush, :bool, :default => false
-
   def emit(tag, es, chain)
     es.each {|time,record|
-      puts "#{Time.at(time).localtime} #{tag}: #{Yajl.dump(record)}"
+      $log.write "#{Time.at(time).localtime} #{tag}: #{Yajl.dump(record)}\n"
     }
-    STDOUT.flush if @autoflush
+    $log.flush
 
     chain.next
   end

data/lib/fluent/supervisor.rb

@@ -20,14 +20,21 @@ module Fluent
 
 class Supervisor
   class LoggerInitializer
-    def initialize(path, level)
+    def initialize(path, level, chuser, chgroup)
       @path = path
       @level = level
+      @chuser = chuser
+      @chgroup = chgroup
     end
 
     def init
       if @path && @path != "-"
         @io = File.open(@path, "a")
+        if @chuser || @chgroup
+          chuid = @chuser ? `id -u #{@chuser}`.to_i : nil
+          chgid = @chgroup ? `id -g #{@chgroup}`.to_i : nil
+          File.chown(chuid, chgid, @path)
+        end
       else
         @io = STDOUT
       end
@@ -61,7 +68,7 @@ class Supervisor
     @plugin_dirs = opt[:plugin_dirs]
     @inline_config = opt[:inline_config]
 
-    @log = LoggerInitializer.new(@log_path, @log_level)
+    @log = LoggerInitializer.new(@log_path, @log_level, @chuser, @chgroup)
     @finished = false
     @main_pid = nil
   end

data/lib/fluent/version.rb

@@ -1,5 +1,5 @@
 module Fluent
 
-VERSION = '0.10.20'
+VERSION = '0.10.21'
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.10.20
+  version: 0.10.21
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-27 00:00:00.000000000Z
+date: 2012-05-01 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
-  requirement: &70174884555560 !ruby/object:Gem::Requirement
+  requirement: &70301065884780 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.4.4
   type: :runtime
   prerelease: false
-  version_requirements: *70174884555560
+  version_requirements: *70301065884780
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70174884536260 !ruby/object:Gem::Requirement
+  requirement: &70301065884240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: 1.4.3
   type: :runtime
   prerelease: false
-  version_requirements: *70174884536260
+  version_requirements: *70301065884240
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
-  requirement: &70174884535660 !ruby/object:Gem::Requirement
+  requirement: &70301065883720 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: '1.0'
   type: :runtime
   prerelease: false
-  version_requirements: *70174884535660
+  version_requirements: *70301065883720
 - !ruby/object:Gem::Dependency
   name: cool.io
-  requirement: &70174884535020 !ruby/object:Gem::Requirement
+  requirement: &70301065883120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 1.1.0
   type: :runtime
   prerelease: false
-  version_requirements: *70174884535020
+  version_requirements: *70301065883120
 - !ruby/object:Gem::Dependency
   name: http_parser.rb
-  requirement: &70174884534420 !ruby/object:Gem::Requirement
+  requirement: &70301065880180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: 0.5.1
   type: :runtime
   prerelease: false
-  version_requirements: *70174884534420
+  version_requirements: *70301065880180
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70174884533820 !ruby/object:Gem::Requirement
+  requirement: &70301065879580 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: 0.9.2
   type: :development
   prerelease: false
-  version_requirements: *70174884533820
+  version_requirements: *70301065879580
 - !ruby/object:Gem::Dependency
   name: rr
-  requirement: &70174884533220 !ruby/object:Gem::Requirement
+  requirement: &70301065878920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *70174884533220
+  version_requirements: *70301065878920
 - !ruby/object:Gem::Dependency
   name: timecop
-  requirement: &70174884532660 !ruby/object:Gem::Requirement
+  requirement: &70301065878300 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: 0.3.0
   type: :development
   prerelease: false
-  version_requirements: *70174884532660
+  version_requirements: *70301065878300
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &70174884532120 !ruby/object:Gem::Requirement
+  requirement: &70301065877720 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,7 +109,7 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *70174884532120
+  version_requirements: *70301065877720
 description: 
 email: frsyuki@gmail.com
 executables: