fluentd 0.10.31 → 0.10.32

data/ChangeLog

@@ -1,4 +1,13 @@
 
+Release 0.10.32 - 2013/03/04
+
+* in_tail: fixed the nil error problem which occurs with pos_file option
+* TextParser/in_tail: supports 'ltsv' (Labeled Tab-Separated Values) format: http://ltsv.org
+* #105: TextParser/in_tail: fixed 'nginx' format
+* #102: show log level in log messages
+* Gemfile: uses https source of rubygems instead of http
+
+
 Release 0.10.31 - 2013/01/31
 
 * out_forward: supports TCP heartbeat

data/VERSION

@@ -1 +1 @@
-0.10.31
+0.10.32

data/lib/fluent/log.rb

@@ -40,13 +40,15 @@ class Log
   LEVEL_ERROR = 4
   LEVEL_FATAL = 5
 
+  LEVEL_TEXT = %w(trace debug info warn error fatal)
+
   def initialize(out=STDERR, level=LEVEL_TRACE)
     @out = out
     @level = level
     @debug_mode = false
     @self_event = false
     @tag = 'fluent'
-    @time_format = '%Y-%m-%d %H:%M:%S %z: '
+    @time_format = '%Y-%m-%d %H:%M:%S %z '
     enable_color out.tty?
   end
 
@@ -95,7 +97,7 @@ class Log
     return if @level > LEVEL_TRACE
     args << block.call if block
     time, msg = event(:trace, args)
-    puts [@color_trace, caller_line(time, 1), msg, @color_reset].join
+    puts [@color_trace, caller_line(time, 1, LEVEL_TRACE), msg, @color_reset].join
   end
   alias TRACE trace
 
@@ -103,7 +105,7 @@ class Log
     return if @level > LEVEL_TRACE
     time = Time.now
     backtrace.each {|msg|
-      puts ["  ", caller_line(time,4), msg].join
+      puts ["  ", caller_line(time, 4, LEVEL_TRACE), msg].join
     }
     nil
   end
@@ -117,7 +119,7 @@ class Log
     return if @level > LEVEL_DEBUG
     args << block.call if block
     time, msg = event(:debug, args)
-    puts [@color_debug, caller_line(time, 1), msg, @color_reset].join
+    puts [@color_debug, caller_line(time, 1, LEVEL_DEBUG), msg, @color_reset].join
   end
   alias DEBUG debug
 
@@ -125,7 +127,7 @@ class Log
     return if @level > LEVEL_DEBUG
     time = Time.now
     backtrace.each {|msg|
-      puts ["  ", caller_line(time,4), msg].join
+      puts ["  ", caller_line(time, 4, LEVEL_DEBUG), msg].join
     }
     nil
   end
@@ -139,7 +141,7 @@ class Log
     return if @level > LEVEL_INFO
     args << block.call if block
     time, msg = event(:info, args)
-    puts [@color_info, caller_line(time, 1), msg, @color_reset].join
+    puts [@color_info, caller_line(time, 1, LEVEL_INFO), msg, @color_reset].join
   end
   alias INFO info
 
@@ -147,7 +149,7 @@ class Log
     return if @level > LEVEL_INFO
     time = Time.now
     backtrace.each {|msg|
-      puts ["  ", caller_line(time,4), msg].join
+      puts ["  ", caller_line(time, 4, LEVEL_INFO), msg].join
     }
     nil
   end
@@ -161,7 +163,7 @@ class Log
     return if @level > LEVEL_WARN
     args << block.call if block
     time, msg = event(:warn, args)
-    puts [@color_warn, caller_line(time, 1), msg, @color_reset].join
+    puts [@color_warn, caller_line(time, 1, LEVEL_WARN), msg, @color_reset].join
   end
   alias WARN warn
 
@@ -169,7 +171,7 @@ class Log
     return if @level > LEVEL_WARN
     time = Time.now
     backtrace.each {|msg|
-      puts ["  ", caller_line(time,4), msg].join
+      puts ["  ", caller_line(time, 4, LEVEL_WARN), msg].join
     }
     nil
   end
@@ -183,7 +185,7 @@ class Log
     return if @level > LEVEL_ERROR
     args << block.call if block
     time, msg = event(:error, args)
-    puts [@color_error, caller_line(time, 1), msg, @color_reset].join
+    puts [@color_error, caller_line(time, 1, LEVEL_ERROR), msg, @color_reset].join
   end
   alias ERROR error
 
@@ -191,7 +193,7 @@ class Log
     return if @level > LEVEL_ERROR
     time = Time.now
     backtrace.each {|msg|
-      puts ["  ", caller_line(time,4), msg].join
+      puts ["  ", caller_line(time, 4, LEVEL_ERROR), msg].join
     }
     nil
   end
@@ -205,7 +207,7 @@ class Log
     return if @level > LEVEL_FATAL
     args << block.call if block
     time, msg = event(:fatal, args)
-    puts [@color_fatal, caller_line(time, 1), msg, @color_reset].join
+    puts [@color_fatal, caller_line(time, 1, LEVEL_FATAL), msg, @color_reset].join
   end
   alias FATAL fatal
 
@@ -213,7 +215,7 @@ class Log
     return if @level > LEVEL_FATAL
     time = Time.now
     backtrace.each {|msg|
-      puts ["  ", caller_line(time,4), msg].join
+      puts ["  ", caller_line(time, 4, LEVEL_FATAL), msg].join
     }
     nil
   end
@@ -266,17 +268,18 @@ class Log
     return time, message
   end
 
-  def caller_line(time, level)
-    line = caller(level+1)[0]
+  def caller_line(time, depth, level)
+    line = caller(depth+1)[0]
+    log_msg = "#{time.strftime(@time_format)}[#{LEVEL_TEXT[level]}]: "
     if @debug_mode
       if match = /^(.+?):(\d+)(?::in `(.*)')?/.match(line)
         file = match[1].split('/')[-2,2].join('/')
         line = match[2]
         method = match[3]
-        return "#{time.strftime(@time_format)}#{file}:#{line}:#{method}: "
+        return "#{log_msg}#{file}:#{line}:#{method}: "
       end
     end
-    return time.strftime(@time_format)
+    return log_msg
   end
 end
 

data/lib/fluent/parser.rb

@@ -136,6 +136,29 @@ class TextParser
     end
   end
 
+  class LabeledTSVParser < ValuesParser
+    config_param :delimiter,       :string, :default => "\t"
+    config_param :label_delimiter, :string, :default =>  ":"
+
+    def configure(conf)
+      conf['keys'] = conf['time_key'] || ''
+      super(conf)
+    end
+
+    def call(text)
+      @keys  = []
+      values = []
+
+      text.split(delimiter).each do |pair|
+        key, value = pair.split(label_delimiter, 2)
+        @keys.push(key)
+        values.push(value)
+      end
+
+      return values_map(values)
+    end
+  end
+
   class CSVParser < ValuesParser
     def initialize
       super
@@ -204,8 +227,9 @@ class TextParser
     'syslog' => Proc.new { RegexpParser.new(/^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?[^\:]*\: *(?<message>.*)$/, {'time_format'=>"%b %d %H:%M:%S"}) },
     'json' => Proc.new { JSONParser.new },
     'tsv' => Proc.new { TSVParser.new },
+    'ltsv' => Proc.new { LabeledTSVParser.new },
     'csv' => Proc.new { CSVParser.new },
-    'nginx' => Proc.new { RegexpParser.new(/^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/,  {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}) },
+    'nginx' => Proc.new { RegexpParser.new(/^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/,  {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}) },
   }
 
   def self.register_template(name, regexp_or_proc, time_format=nil)

data/lib/fluent/plugin/in_tail.rb

@@ -459,12 +459,14 @@ class TailInput < Input
 
     def read_inode
       @file.pos = @seek + INO_OFFSET
-      @file.read(8).to_i(16)
+      raw = @file.read(8)
+      raw ? raw.to_i(16) : 0
     end
 
     def read_pos
       @file.pos = @seek
-      @file.read(16).to_i(16)
+      raw = @file.read(16)
+      raw ? raw.to_i(16) : 0
     end
   end
 

data/lib/fluent/plugin/out_forward.rb

@@ -128,14 +128,6 @@ class ForwardOutput < ObjectBufferedOutput
     $log.error_backtrace
   end
 
-  # override BufferedOutput#emit
-  def emit(tag, es, chain)
-    data = es.to_msgpack_stream
-    if @buffer.emit(tag, data, chain)  # use key = tag
-      submit_flush
-    end
-  end
-
   def write_objects(tag, es)
     error = nil
 

data/lib/fluent/status.rb

@@ -22,20 +22,25 @@ module Fluent
 class StatusClass
   def initialize
     @entries = {}
+    @mutex = Mutex.new
   end
 
   def register(instance, name, &block)
-    (@entries[instance.object_id] ||= {})[name] = block
+    @mutex.synchronize {
+      (@entries[instance.object_id] ||= {})[name] = block
+    }
     nil
   end
 
   def each(&block)
-    @entries.each {|obj_id,hash|
-      record = {}
-      hash.each_pair {|name,block|
-        record[name] = block.call
+    @mutex.synchronize {
+      @entries.each {|obj_id,hash|
+        record = {}
+        hash.each_pair {|name,block|
+          record[name] = block.call
+        }
+        block.call(record)
       }
-      block.call(record)
     }
   end
 end

data/lib/fluent/version.rb

@@ -1,5 +1,5 @@
 module Fluent
 
-VERSION = '0.10.31'
+VERSION = '0.10.32'
 
 end

data/test/parser.rb

@@ -0,0 +1,193 @@
+require 'helper'
+require 'fluent/test'
+require 'fluent/parser'
+
+module ParserTest
+  include Fluent
+
+  def str2time(str_time, format = nil)
+    if format
+      Time.strptime(str_time, format).to_i
+    else
+      Time.parse(str_time)
+    end
+  end
+
+  class ApacheParserTest < ::Test::Unit::TestCase
+    include ParserTest
+
+    def setup
+      @parser = TextParser::TEMPLATE_FACTORIES['apache'].call
+    end
+
+    def test_call
+      time, record = @parser.call('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777')
+
+      assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
+      assert_equal({
+        'user'    => '-',
+        'method'  => 'GET',
+        'code'    => '200',
+        'size'    => '777',
+        'host'    => '192.168.0.1',
+        'path'    => '/'
+      }, record)
+    end
+  end
+
+  class Apache2ParserTest < ::Test::Unit::TestCase
+    include ParserTest
+
+    def setup
+      @parser = TextParser::ApacheParser.new
+    end
+
+    def test_call
+      time, record = @parser.call('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"')
+
+      assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
+      assert_equal({
+        'user'    => nil,
+        'method'  => 'GET',
+        'code'    => 200,
+        'size'    => 777,
+        'host'    => '192.168.0.1',
+        'path'    => '/',
+        'referer' => nil,
+        'agent'   => 'Opera/12.0'
+      }, record)
+    end
+  end
+
+  class SyslogParserTest < ::Test::Unit::TestCase
+    include ParserTest
+
+    def setup
+      @parser = TextParser::TEMPLATE_FACTORIES['syslog'].call
+    end
+
+    def test_call
+      time, record = @parser.call('Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test')
+
+      assert_equal(str2time('Feb 28 12:00:00', '%b %d %H:%M:%S'), time)
+      assert_equal({
+        'host'    => '192.168.0.1',
+        'ident'   => 'fluentd',
+        'pid'     => '11111',
+        'message' => '[error] Syslog test'
+      }, record)
+    end
+  end
+
+  class JsonParserTest < ::Test::Unit::TestCase
+    include ParserTest
+
+    def setup
+      @parser = TextParser::JSONParser.new
+    end
+
+    def test_call
+      time, record = @parser.call('{"time":1362020400,"host":"192.168.0.1","size":777,"method":"PUT"}')
+
+      assert_equal(str2time('2013-02-28 12:00:00 +0900').to_i, time)
+      assert_equal({
+        'host'   => '192.168.0.1',
+        'size'   => 777,
+        'method' => 'PUT',
+      }, record)
+    end
+  end
+
+  class NginxParserTest < ::Test::Unit::TestCase
+    include ParserTest
+
+    def setup
+      @parser = TextParser::TEMPLATE_FACTORIES['nginx'].call
+      @expected = {
+        'remote'  => '127.0.0.1',
+        'host'    => '192.168.0.1',
+        'user'    => '-',
+        'method'  => 'GET',
+        'path'    => '/',
+        'code'    => '200',
+        'size'    => '777',
+        'referer' => '-',
+        'agent'   => 'Opera/12.0'
+      }
+    end
+
+    def test_call
+      time, record = @parser.call('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"')
+
+      assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
+      assert_equal(@expected, record)
+    end
+
+    def test_call_with_empty_included_path
+      time, record = @parser.call('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET /a[ ]b HTTP/1.1" 200 777 "-" "Opera/12.0"')
+
+      assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
+      assert_equal(@expected.merge('path' => '/a[ ]b'), record)
+    end
+  end
+
+  class LabeledTSVParserTest < ::Test::Unit::TestCase
+    include ParserTest
+
+    def test_config_params
+      parser = TextParser::LabeledTSVParser.new
+
+      assert_equal "\t", parser.delimiter
+      assert_equal  ":", parser.label_delimiter
+
+      parser.configure(
+        'delimiter'       => ',',
+        'label_delimiter' => '=',
+      )
+
+      assert_equal ",", parser.delimiter
+      assert_equal "=", parser.label_delimiter
+    end
+
+    def test_call
+      parser = TextParser::LabeledTSVParser.new
+      time, record = parser.call("time:[28/Feb/2013:12:00:00 +0900]\thost:192.168.0.1\treq:GET /list HTTP/1.1")
+
+      assert_equal({
+        'time' => '[28/Feb/2013:12:00:00 +0900]',
+        'host' => '192.168.0.1',
+        'req'  => 'GET /list HTTP/1.1',
+      }, record)
+    end
+
+    def test_call_with_customized_delimiter
+      parser = TextParser::LabeledTSVParser.new
+      parser.configure(
+        'delimiter'       => ',',
+        'label_delimiter' => '=',
+      )
+      time, record = parser.call('time=[28/Feb/2013:12:00:00 +0900],host=192.168.0.1,req=GET /list HTTP/1.1')
+
+      assert_equal({
+        'time' => '[28/Feb/2013:12:00:00 +0900]',
+        'host' => '192.168.0.1',
+        'req'  => 'GET /list HTTP/1.1',
+      }, record)
+    end
+
+    def test_call_with_customized_time_format
+      parser = TextParser::LabeledTSVParser.new
+      parser.configure(
+        'time_key'    => 'time',
+        'time_format' => '[%d/%b/%Y:%H:%M:%S %z]',
+      )
+      time, record = parser.call("time:[28/Feb/2013:12:00:00 +0900]\thost:192.168.0.1\treq:GET /list HTTP/1.1")
+
+      assert_equal({
+        'time' => '[28/Feb/2013:12:00:00 +0900]',
+        'host' => '192.168.0.1',
+        'req'  => 'GET /list HTTP/1.1',
+      }, record)
+    end
+  end
+end

data/test/plugin/in_gc_stat.rb

@@ -0,0 +1,36 @@
+require 'fluent/test'
+
+class ObjectSpaceInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    emit_interval 1
+    tag t1
+  ]
+
+  def create_driver(conf=CONFIG)
+    Fluent::Test::InputTestDriver.new(Fluent::GCStatInput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal(1, d.instance.emit_interval)
+    assert_equal("t1", d.instance.tag)
+  end
+
+  def test_emit
+    stat = GC.stat
+    stub(GC).stat { stat }
+
+    d = create_driver
+    d.run do
+      sleep 2
+    end
+
+    emits = d.emits
+    assert(emits.length > 0)
+    assert_equal(stat, emits[0][2])
+  end
+end

data/test/plugin/in_status.rb

@@ -0,0 +1,37 @@
+require 'fluent/test'
+
+class StatusInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    emit_interval 1
+    tag t1
+  ]
+
+  def create_driver(conf=CONFIG)
+    Fluent::Test::InputTestDriver.new(Fluent::StatusInput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal(1, d.instance.emit_interval)
+    assert_equal("t1", d.instance.tag)
+  end
+
+  def test_emit
+    stub(Fluent::Status).each { |b|
+      b.call("answer" => "42")
+    }
+
+    d = create_driver
+    d.run do
+      sleep 2
+    end
+
+    emits = d.emits
+    assert(emits.length > 0)
+    assert_equal({"answer" => "42"}, emits[0][2])
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.10.31
+  version: 0.10.32
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-31 00:00:00.000000000 Z
+date: 2013-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -229,10 +229,13 @@ files:
 - test/helper.rb
 - test/match.rb
 - test/mixin.rb
+- test/parser.rb
 - test/plugin/in_exec.rb
 - test/plugin/in_forward.rb
+- test/plugin/in_gc_stat.rb
 - test/plugin/in_http.rb
 - test/plugin/in_object_space.rb
+- test/plugin/in_status.rb
 - test/plugin/in_stream.rb
 - test/plugin/in_syslog.rb
 - test/plugin/in_tail.rb
@@ -275,10 +278,13 @@ test_files:
 - test/helper.rb
 - test/match.rb
 - test/mixin.rb
+- test/parser.rb
 - test/plugin/in_exec.rb
 - test/plugin/in_forward.rb
+- test/plugin/in_gc_stat.rb
 - test/plugin/in_http.rb
 - test/plugin/in_object_space.rb
+- test/plugin/in_status.rb
 - test/plugin/in_stream.rb
 - test/plugin/in_syslog.rb
 - test/plugin/in_tail.rb